From fbf983c2ff88c339bf6ed962d6aebe37d9cfb31b Mon Sep 17 00:00:00 2001 From: James Allen Date: Fri, 7 Nov 2014 17:38:12 +0000 Subject: [PATCH 001/491] Create framework for real-time API with session authentication --- services/real-time/.gitignore | 5 + services/real-time/Gruntfile.coffee | 74 +++ services/real-time/app.coffee | 50 ++ services/real-time/app/coffee/Router.coffee | 20 + services/real-time/client.coffee | 0 .../real-time/config/settings.defaults.coffee | 16 + services/real-time/package.json | 39 ++ .../acceptance/coffee/SessionTests.coffee | 47 ++ .../coffee/helpers/RealTimeClient.coffee | 40 ++ .../test/acceptance/libs/XMLHttpRequest.js | 548 ++++++++++++++++++ 10 files changed, 839 insertions(+) create mode 100644 services/real-time/.gitignore create mode 100644 services/real-time/Gruntfile.coffee create mode 100644 services/real-time/app.coffee create mode 100644 services/real-time/app/coffee/Router.coffee create mode 100644 services/real-time/client.coffee create mode 100644 services/real-time/config/settings.defaults.coffee create mode 100644 services/real-time/package.json create mode 100644 services/real-time/test/acceptance/coffee/SessionTests.coffee create mode 100644 services/real-time/test/acceptance/coffee/helpers/RealTimeClient.coffee create mode 100644 services/real-time/test/acceptance/libs/XMLHttpRequest.js diff --git a/services/real-time/.gitignore b/services/real-time/.gitignore new file mode 100644 index 0000000000..e3b29a58ff --- /dev/null +++ b/services/real-time/.gitignore @@ -0,0 +1,5 @@ +node_modules +app.js +app/js +test/unit/js +test/acceptance/js \ No newline at end of file diff --git a/services/real-time/Gruntfile.coffee b/services/real-time/Gruntfile.coffee new file mode 100644 index 0000000000..e01c8aecb1 --- /dev/null +++ b/services/real-time/Gruntfile.coffee @@ -0,0 +1,74 @@ +module.exports = (grunt) -> + grunt.initConfig + forever: + app: + options: + index: "app.js" + coffee: + app_src: + expand: true, + flatten: true, + cwd: "app" + src: ['coffee/*.coffee'], + dest: 'app/js/', + ext: '.js' + + app: + src: "app.coffee" + dest: "app.js" + + unit_tests: + expand: true + cwd: "test/unit/coffee" + src: ["**/*.coffee"] + dest: "test/unit/js/" + ext: ".js" + + acceptance_tests: + expand: true + cwd: "test/acceptance/coffee" + src: ["**/*.coffee"] + dest: "test/acceptance/js/" + ext: ".js" + clean: + app: ["app/js/"] + unit_tests: ["test/unit/js"] + acceptance_tests: ["test/acceptance/js"] + smoke_tests: ["test/smoke/js"] + + execute: + app: + src: "app.js" + + mochaTest: + unit: + options: + reporter: grunt.option('reporter') or 'spec' + src: ["test/unit/js/**/*.js"] + acceptance: + options: + reporter: grunt.option('reporter') or 'spec' + timeout: 40000 + grep: grunt.option("grep") + src: ["test/acceptance/js/**/*.js"] + + grunt.loadNpmTasks 'grunt-contrib-coffee' + grunt.loadNpmTasks 'grunt-contrib-clean' + grunt.loadNpmTasks 'grunt-mocha-test' + grunt.loadNpmTasks 'grunt-shell' + grunt.loadNpmTasks 'grunt-execute' + grunt.loadNpmTasks 'grunt-bunyan' + grunt.loadNpmTasks 'grunt-forever' + + grunt.registerTask 'compile:app', ['clean:app', 'coffee:app', 'coffee:app_src'] + grunt.registerTask 'run', ['compile:app', 'bunyan', 'execute'] + + grunt.registerTask 'compile:unit_tests', ['clean:unit_tests', 'coffee:unit_tests'] + grunt.registerTask 'test:unit', ['compile:app', 'compile:unit_tests', 'mochaTest:unit'] + + grunt.registerTask 'compile:acceptance_tests', ['clean:acceptance_tests', 'coffee:acceptance_tests'] + grunt.registerTask 'test:acceptance', ['compile:acceptance_tests', 'mochaTest:acceptance'] + + grunt.registerTask 'install', 'compile:app' + + grunt.registerTask 'default', ['run'] \ No newline at end of file diff --git a/services/real-time/app.coffee b/services/real-time/app.coffee new file mode 100644 index 0000000000..15097f51b8 --- /dev/null +++ b/services/real-time/app.coffee @@ -0,0 +1,50 @@ +express = require("express") +session = require("express-session") +redis = require("redis-sharelatex") +RedisStore = require('connect-redis')(session) +SessionSockets = require('session.socket.io') +CookieParser = require("cookie-parser") + +Settings = require "settings-sharelatex" + +logger = require "logger-sharelatex" +logger.initialize("real-time-sharelatex") + +Metrics = require("metrics-sharelatex") +Metrics.initialize("real-time") + +rclient = redis.createClient(Settings.redis.web) + +# Set up socket.io server +app = express() +server = require('http').createServer(app) +io = require('socket.io').listen(server) + +# Bind to sessions +sessionStore = new RedisStore(client: rclient) +cookieParser = CookieParser(Settings.security.sessionSecret) +sessionSockets = new SessionSockets(io, sessionStore, cookieParser, Settings.cookieName) + +io.configure -> + io.enable('browser client minification') + io.enable('browser client etag') + + # Fix for Safari 5 error of "Error during WebSocket handshake: location mismatch" + # See http://answers.dotcloud.com/question/578/problem-with-websocket-over-ssl-in-safari-with + io.set('match origin protocol', true) + + # gzip uses a Node 0.8.x method of calling the gzip program which + # doesn't work with 0.6.x + #io.enable('browser client gzip') + io.set('transports', ['websocket', 'flashsocket', 'htmlfile', 'xhr-polling', 'jsonp-polling']) + io.set('log level', 1) + +Router = require "./app/js/Router" +Router.configure(app, io, sessionSockets) + +port = Settings.internal.realTime.port +host = Settings.internal.realTime.host + +server.listen port, host, (error) -> + throw error if error? + logger.log "real-time-sharelatex listening on #{host}:#{port}" \ No newline at end of file diff --git a/services/real-time/app/coffee/Router.coffee b/services/real-time/app/coffee/Router.coffee new file mode 100644 index 0000000000..fc363d30ee --- /dev/null +++ b/services/real-time/app/coffee/Router.coffee @@ -0,0 +1,20 @@ +Metrics = require "metrics-sharelatex" +logger = require "logger-sharelatex" + +module.exports = Router = + configure: (app, io, session) -> + session.on 'connection', (error, client, session) -> + if error? + logger.err err: error, "error when client connected" + client?.disconnect() + return + + Metrics.inc('socket-io.connection') + + logger.log session: session, "got session" + + user = session.user + if !user? + logger.log "terminating session without authenticated user" + client.disconnect() + return \ No newline at end of file diff --git a/services/real-time/client.coffee b/services/real-time/client.coffee new file mode 100644 index 0000000000..e69de29bb2 diff --git a/services/real-time/config/settings.defaults.coffee b/services/real-time/config/settings.defaults.coffee new file mode 100644 index 0000000000..7b069975b1 --- /dev/null +++ b/services/real-time/config/settings.defaults.coffee @@ -0,0 +1,16 @@ +module.exports = + redis: + web: + host: "localhost" + port: "6379" + password: "" + + internal: + realTime: + port: 3026 + host: "localhost" + + security: + sessionSecret: "secret-please-change" + + cookieName:"sharelatex.sid" \ No newline at end of file diff --git a/services/real-time/package.json b/services/real-time/package.json new file mode 100644 index 0000000000..01b664e945 --- /dev/null +++ b/services/real-time/package.json @@ -0,0 +1,39 @@ +{ + "name": "real-time-sharelatex", + "version": "0.0.1", + "description": "The socket.io layer of ShareLaTeX for real-time editor interactions", + "author": "ShareLaTeX ", + "repository": { + "type": "git", + "url": "https://github.com/sharelatex/real-time-sharelatex.git" + }, + "dependencies": { + "connect-redis": "^2.1.0", + "express": "^4.10.1", + "express-session": "^1.9.1", + "logger-sharelatex": "git+https://github.com/sharelatex/logger-sharelatex.git#v1.0.0", + "metrics-sharelatex": "git+https://github.com/sharelatex/metrics-sharelatex.git#v1.0.0", + "redis-sharelatex": "~0.0.4", + "session.socket.io": "^0.1.6", + "settings-sharelatex": "git+https://github.com/sharelatex/settings-sharelatex.git#v1.0.0", + "socket.io": "0.9.16", + "socket.io-client": "^0.9.16" + }, + "devDependencies": { + "bunyan": "~0.22.3", + "chai": "~1.9.1", + "cookie-signature": "^1.0.5", + "grunt": "~0.4.4", + "grunt-bunyan": "~0.5.0", + "grunt-contrib-clean": "~0.5.0", + "grunt-contrib-coffee": "~0.10.1", + "grunt-execute": "~0.2.1", + "grunt-forever": "~0.4.4", + "grunt-mocha-test": "~0.10.2", + "grunt-shell": "~0.7.0", + "request": "~2.34.0", + "sandboxed-module": "~0.3.0", + "sinon": "~1.5.2", + "uid-safe": "^1.0.1" + } +} diff --git a/services/real-time/test/acceptance/coffee/SessionTests.coffee b/services/real-time/test/acceptance/coffee/SessionTests.coffee new file mode 100644 index 0000000000..01b31cee01 --- /dev/null +++ b/services/real-time/test/acceptance/coffee/SessionTests.coffee @@ -0,0 +1,47 @@ +chai = require("chai") +expect = chai.expect + +RealTimeClient = require "./helpers/RealTimeClient" + +describe "Session", -> + describe "with an established session", -> + beforeEach (done) -> + RealTimeClient.setSession { + user: { _id: @user_id } + }, (error) => + throw error if error? + @client = RealTimeClient.connect() + done() + + it "should not get disconnected", (done) -> + disconnected = false + @client.on "disconnect", () -> + disconnected = true + setTimeout () => + expect(disconnected).to.equal false + done() + , 500 + + describe "without an established session", -> + beforeEach (done) -> + RealTimeClient.unsetSession (error) => + throw error if error? + @client = RealTimeClient.connect() + done() + + it "should get disconnected", (done) -> + @client.on "disconnect", () -> + done() + + describe "with a user set on the session", -> + beforeEach (done) -> + RealTimeClient.setSession { + foo: "bar" + }, (error) => + throw error if error? + @client = RealTimeClient.connect() + done() + + it "should get disconnected", (done) -> + @client.on "disconnect", () -> + done() \ No newline at end of file diff --git a/services/real-time/test/acceptance/coffee/helpers/RealTimeClient.coffee b/services/real-time/test/acceptance/coffee/helpers/RealTimeClient.coffee new file mode 100644 index 0000000000..415f14fe5e --- /dev/null +++ b/services/real-time/test/acceptance/coffee/helpers/RealTimeClient.coffee @@ -0,0 +1,40 @@ +XMLHttpRequest = require("../../libs/XMLHttpRequest").XMLHttpRequest +io = require("socket.io-client") + +Settings = require "settings-sharelatex" +redis = require "redis-sharelatex" +rclient = redis.createClient(Settings.redis.web) + +uid = require('uid-safe').sync +signature = require("cookie-signature") + +io.util.request = () -> + xhr = new XMLHttpRequest() + _open = xhr.open + xhr.open = () => + _open.apply(xhr, arguments) + if Client.cookie? + xhr.setRequestHeader("Cookie", Client.cookie) + return xhr + +module.exports = Client = + cookie: null + + setSession: (session, callback = (error) ->) -> + sessionId = uid(24) + session.cookie = {} + rclient.set "sess:" + sessionId, JSON.stringify(session), (error) -> + return callback(error) if error? + secret = Settings.security.sessionSecret + cookieKey = 's:' + signature.sign(sessionId, secret) + Client.cookie = "#{Settings.cookieName}=#{cookieKey}" + callback() + + unsetSession: (callback = (error) ->) -> + Client.cookie = null + callback() + + connect: (cookie) -> + client = io.connect("http://localhost:3026", 'force new connection': true) + return client + diff --git a/services/real-time/test/acceptance/libs/XMLHttpRequest.js b/services/real-time/test/acceptance/libs/XMLHttpRequest.js new file mode 100644 index 0000000000..e79634da5e --- /dev/null +++ b/services/real-time/test/acceptance/libs/XMLHttpRequest.js @@ -0,0 +1,548 @@ +/** + * Wrapper for built-in http.js to emulate the browser XMLHttpRequest object. + * + * This can be used with JS designed for browsers to improve reuse of code and + * allow the use of existing libraries. + * + * Usage: include("XMLHttpRequest.js") and use XMLHttpRequest per W3C specs. + * + * @author Dan DeFelippi + * @contributor David Ellis + * @license MIT + */ + +var Url = require("url") + , spawn = require("child_process").spawn + , fs = require('fs'); + +exports.XMLHttpRequest = function() { + /** + * Private variables + */ + var self = this; + var http = require('http'); + var https = require('https'); + + // Holds http.js objects + var client; + var request; + var response; + + // Request settings + var settings = {}; + + // Set some default headers + var defaultHeaders = { + "User-Agent": "node-XMLHttpRequest", + "Accept": "*/*", + }; + + var headers = defaultHeaders; + + // These headers are not user setable. + // The following are allowed but banned in the spec: + // * user-agent + var forbiddenRequestHeaders = [ + "accept-charset", + "accept-encoding", + "access-control-request-headers", + "access-control-request-method", + "connection", + "content-length", + "content-transfer-encoding", + //"cookie", + "cookie2", + "date", + "expect", + "host", + "keep-alive", + "origin", + "referer", + "te", + "trailer", + "transfer-encoding", + "upgrade", + "via" + ]; + + // These request methods are not allowed + var forbiddenRequestMethods = [ + "TRACE", + "TRACK", + "CONNECT" + ]; + + // Send flag + var sendFlag = false; + // Error flag, used when errors occur or abort is called + var errorFlag = false; + + // Event listeners + var listeners = {}; + + /** + * Constants + */ + + this.UNSENT = 0; + this.OPENED = 1; + this.HEADERS_RECEIVED = 2; + this.LOADING = 3; + this.DONE = 4; + + /** + * Public vars + */ + + // Current state + this.readyState = this.UNSENT; + + // default ready state change handler in case one is not set or is set late + this.onreadystatechange = null; + + // Result & response + this.responseText = ""; + this.responseXML = ""; + this.status = null; + this.statusText = null; + + /** + * Private methods + */ + + /** + * Check if the specified header is allowed. + * + * @param string header Header to validate + * @return boolean False if not allowed, otherwise true + */ + var isAllowedHttpHeader = function(header) { + return (header && forbiddenRequestHeaders.indexOf(header.toLowerCase()) === -1); + }; + + /** + * Check if the specified method is allowed. + * + * @param string method Request method to validate + * @return boolean False if not allowed, otherwise true + */ + var isAllowedHttpMethod = function(method) { + return (method && forbiddenRequestMethods.indexOf(method) === -1); + }; + + /** + * Public methods + */ + + /** + * Open the connection. Currently supports local server requests. + * + * @param string method Connection method (eg GET, POST) + * @param string url URL for the connection. + * @param boolean async Asynchronous connection. Default is true. + * @param string user Username for basic authentication (optional) + * @param string password Password for basic authentication (optional) + */ + this.open = function(method, url, async, user, password) { + this.abort(); + errorFlag = false; + + // Check for valid request method + if (!isAllowedHttpMethod(method)) { + throw "SecurityError: Request method not allowed"; + return; + } + + settings = { + "method": method, + "url": url.toString(), + "async": (typeof async !== "boolean" ? true : async), + "user": user || null, + "password": password || null + }; + + setState(this.OPENED); + }; + + /** + * Sets a header for the request. + * + * @param string header Header name + * @param string value Header value + */ + this.setRequestHeader = function(header, value) { + if (this.readyState != this.OPENED) { + throw "INVALID_STATE_ERR: setRequestHeader can only be called when state is OPEN"; + } + if (!isAllowedHttpHeader(header)) { + console.warn('Refused to set unsafe header "' + header + '"'); + return; + } + if (sendFlag) { + throw "INVALID_STATE_ERR: send flag is true"; + } + headers[header] = value; + }; + + /** + * Gets a header from the server response. + * + * @param string header Name of header to get. + * @return string Text of the header or null if it doesn't exist. + */ + this.getResponseHeader = function(header) { + if (typeof header === "string" + && this.readyState > this.OPENED + && response.headers[header.toLowerCase()] + && !errorFlag + ) { + return response.headers[header.toLowerCase()]; + } + + return null; + }; + + /** + * Gets all the response headers. + * + * @return string A string with all response headers separated by CR+LF + */ + this.getAllResponseHeaders = function() { + if (this.readyState < this.HEADERS_RECEIVED || errorFlag) { + return ""; + } + var result = ""; + + for (var i in response.headers) { + // Cookie headers are excluded + if (i !== "set-cookie" && i !== "set-cookie2") { + result += i + ": " + response.headers[i] + "\r\n"; + } + } + return result.substr(0, result.length - 2); + }; + + /** + * Gets a request header + * + * @param string name Name of header to get + * @return string Returns the request header or empty string if not set + */ + this.getRequestHeader = function(name) { + // @TODO Make this case insensitive + if (typeof name === "string" && headers[name]) { + return headers[name]; + } + + return ""; + } + + /** + * Sends the request to the server. + * + * @param string data Optional data to send as request body. + */ + this.send = function(data) { + if (this.readyState != this.OPENED) { + throw "INVALID_STATE_ERR: connection must be opened before send() is called"; + } + + if (sendFlag) { + throw "INVALID_STATE_ERR: send has already been called"; + } + + var ssl = false, local = false; + var url = Url.parse(settings.url); + + // Determine the server + switch (url.protocol) { + case 'https:': + ssl = true; + // SSL & non-SSL both need host, no break here. + case 'http:': + var host = url.hostname; + break; + + case 'file:': + local = true; + break; + + case undefined: + case '': + var host = "localhost"; + break; + + default: + throw "Protocol not supported."; + } + + // Load files off the local filesystem (file://) + if (local) { + if (settings.method !== "GET") { + throw "XMLHttpRequest: Only GET method is supported"; + } + + if (settings.async) { + fs.readFile(url.pathname, 'utf8', function(error, data) { + if (error) { + self.handleError(error); + } else { + self.status = 200; + self.responseText = data; + setState(self.DONE); + } + }); + } else { + try { + this.responseText = fs.readFileSync(url.pathname, 'utf8'); + this.status = 200; + setState(self.DONE); + } catch(e) { + this.handleError(e); + } + } + + return; + } + + // Default to port 80. If accessing localhost on another port be sure + // to use http://localhost:port/path + var port = url.port || (ssl ? 443 : 80); + // Add query string if one is used + var uri = url.pathname + (url.search ? url.search : ''); + + // Set the Host header or the server may reject the request + headers["Host"] = host; + if (!((ssl && port === 443) || port === 80)) { + headers["Host"] += ':' + url.port; + } + + // Set Basic Auth if necessary + if (settings.user) { + if (typeof settings.password == "undefined") { + settings.password = ""; + } + var authBuf = new Buffer(settings.user + ":" + settings.password); + headers["Authorization"] = "Basic " + authBuf.toString("base64"); + } + + // Set content length header + if (settings.method === "GET" || settings.method === "HEAD") { + data = null; + } else if (data) { + headers["Content-Length"] = Buffer.byteLength(data); + + if (!headers["Content-Type"]) { + headers["Content-Type"] = "text/plain;charset=UTF-8"; + } + } else if (settings.method === "POST") { + // For a post with no data set Content-Length: 0. + // This is required by buggy servers that don't meet the specs. + headers["Content-Length"] = 0; + } + + var options = { + host: host, + port: port, + path: uri, + method: settings.method, + headers: headers + }; + + // Reset error flag + errorFlag = false; + + // Handle async requests + if (settings.async) { + // Use the proper protocol + var doRequest = ssl ? https.request : http.request; + + // Request is being sent, set send flag + sendFlag = true; + + // As per spec, this is called here for historical reasons. + self.dispatchEvent("readystatechange"); + + // Create the request + request = doRequest(options, function(resp) { + response = resp; + response.setEncoding("utf8"); + + setState(self.HEADERS_RECEIVED); + self.status = response.statusCode; + + response.on('data', function(chunk) { + // Make sure there's some data + if (chunk) { + self.responseText += chunk; + } + // Don't emit state changes if the connection has been aborted. + if (sendFlag) { + setState(self.LOADING); + } + }); + + response.on('end', function() { + if (sendFlag) { + // Discard the 'end' event if the connection has been aborted + setState(self.DONE); + sendFlag = false; + } + }); + + response.on('error', function(error) { + self.handleError(error); + }); + }).on('error', function(error) { + self.handleError(error); + }); + + // Node 0.4 and later won't accept empty data. Make sure it's needed. + if (data) { + request.write(data); + } + + request.end(); + + self.dispatchEvent("loadstart"); + } else { // Synchronous + // Create a temporary file for communication with the other Node process + var syncFile = ".node-xmlhttprequest-sync-" + process.pid; + fs.writeFileSync(syncFile, "", "utf8"); + // The async request the other Node process executes + var execString = "var http = require('http'), https = require('https'), fs = require('fs');" + + "var doRequest = http" + (ssl ? "s" : "") + ".request;" + + "var options = " + JSON.stringify(options) + ";" + + "var responseText = '';" + + "var req = doRequest(options, function(response) {" + + "response.setEncoding('utf8');" + + "response.on('data', function(chunk) {" + + "responseText += chunk;" + + "});" + + "response.on('end', function() {" + + "fs.writeFileSync('" + syncFile + "', 'NODE-XMLHTTPREQUEST-STATUS:' + response.statusCode + ',' + responseText, 'utf8');" + + "});" + + "response.on('error', function(error) {" + + "fs.writeFileSync('" + syncFile + "', 'NODE-XMLHTTPREQUEST-ERROR:' + JSON.stringify(error), 'utf8');" + + "});" + + "}).on('error', function(error) {" + + "fs.writeFileSync('" + syncFile + "', 'NODE-XMLHTTPREQUEST-ERROR:' + JSON.stringify(error), 'utf8');" + + "});" + + (data ? "req.write('" + data.replace(/'/g, "\\'") + "');":"") + + "req.end();"; + // Start the other Node Process, executing this string + syncProc = spawn(process.argv[0], ["-e", execString]); + while((self.responseText = fs.readFileSync(syncFile, 'utf8')) == "") { + // Wait while the file is empty + } + // Kill the child process once the file has data + syncProc.stdin.end(); + // Remove the temporary file + fs.unlinkSync(syncFile); + if (self.responseText.match(/^NODE-XMLHTTPREQUEST-ERROR:/)) { + // If the file returned an error, handle it + var errorObj = self.responseText.replace(/^NODE-XMLHTTPREQUEST-ERROR:/, ""); + self.handleError(errorObj); + } else { + // If the file returned okay, parse its data and move to the DONE state + self.status = self.responseText.replace(/^NODE-XMLHTTPREQUEST-STATUS:([0-9]*),.*/, "$1"); + self.responseText = self.responseText.replace(/^NODE-XMLHTTPREQUEST-STATUS:[0-9]*,(.*)/, "$1"); + setState(self.DONE); + } + } + }; + + /** + * Called when an error is encountered to deal with it. + */ + this.handleError = function(error) { + this.status = 503; + this.statusText = error; + this.responseText = error.stack; + errorFlag = true; + setState(this.DONE); + }; + + /** + * Aborts a request. + */ + this.abort = function() { + if (request) { + request.abort(); + request = null; + } + + headers = defaultHeaders; + this.responseText = ""; + this.responseXML = ""; + + errorFlag = true; + + if (this.readyState !== this.UNSENT + && (this.readyState !== this.OPENED || sendFlag) + && this.readyState !== this.DONE) { + sendFlag = false; + setState(this.DONE); + } + this.readyState = this.UNSENT; + }; + + /** + * Adds an event listener. Preferred method of binding to events. + */ + this.addEventListener = function(event, callback) { + if (!(event in listeners)) { + listeners[event] = []; + } + // Currently allows duplicate callbacks. Should it? + listeners[event].push(callback); + }; + + /** + * Remove an event callback that has already been bound. + * Only works on the matching funciton, cannot be a copy. + */ + this.removeEventListener = function(event, callback) { + if (event in listeners) { + // Filter will return a new array with the callback removed + listeners[event] = listeners[event].filter(function(ev) { + return ev !== callback; + }); + } + }; + + /** + * Dispatch any events, including both "on" methods and events attached using addEventListener. + */ + this.dispatchEvent = function(event) { + if (typeof self["on" + event] === "function") { + self["on" + event](); + } + if (event in listeners) { + for (var i = 0, len = listeners[event].length; i < len; i++) { + listeners[event][i].call(self); + } + } + }; + + /** + * Changes readyState and calls onreadystatechange. + * + * @param int state New state + */ + var setState = function(state) { + if (self.readyState !== state) { + self.readyState = state; + + if (settings.async || self.readyState < self.OPENED || self.readyState === self.DONE) { + self.dispatchEvent("readystatechange"); + } + + if (self.readyState === self.DONE && !errorFlag) { + self.dispatchEvent("load"); + // @TODO figure out InspectorInstrumentation::didLoadXHR(cookie) + self.dispatchEvent("loadend"); + } + } + }; +}; From 02c0a3a867dfff2ac59bfe0dec61c9bcea46b804 Mon Sep 17 00:00:00 2001 From: James Allen Date: Mon, 10 Nov 2014 11:27:08 +0000 Subject: [PATCH 002/491] Create joinProject socket.io endpoint --- services/real-time/app/coffee/Router.coffee | 9 +- .../real-time/app/coffee/WebApiManager.coffee | 21 +++++ .../app/coffee/WebsocketController.coffee | 31 +++++++ .../real-time/config/settings.defaults.coffee | 4 + services/real-time/package.json | 3 +- .../acceptance/coffee/JoinProjectTests.coffee | 44 +++++++++ .../acceptance/coffee/SessionTests.coffee | 3 +- .../coffee/helpers/MockWebClient.coffee | 39 ++++++++ .../unit/coffee/WebApiManagerTests.coffee | 55 ++++++++++++ .../coffee/WebsocketControllerTests.coffee | 89 +++++++++++++++++++ 10 files changed, 294 insertions(+), 4 deletions(-) create mode 100644 services/real-time/app/coffee/WebApiManager.coffee create mode 100644 services/real-time/app/coffee/WebsocketController.coffee create mode 100644 services/real-time/test/acceptance/coffee/JoinProjectTests.coffee create mode 100644 services/real-time/test/acceptance/coffee/helpers/MockWebClient.coffee create mode 100644 services/real-time/test/unit/coffee/WebApiManagerTests.coffee create mode 100644 services/real-time/test/unit/coffee/WebsocketControllerTests.coffee diff --git a/services/real-time/app/coffee/Router.coffee b/services/real-time/app/coffee/Router.coffee index fc363d30ee..73c3aa0acd 100644 --- a/services/real-time/app/coffee/Router.coffee +++ b/services/real-time/app/coffee/Router.coffee @@ -1,5 +1,6 @@ Metrics = require "metrics-sharelatex" logger = require "logger-sharelatex" +WebsocketController = require "./WebsocketController" module.exports = Router = configure: (app, io, session) -> @@ -14,7 +15,11 @@ module.exports = Router = logger.log session: session, "got session" user = session.user - if !user? + if !user? or !user._id? logger.log "terminating session without authenticated user" client.disconnect() - return \ No newline at end of file + return + + client.on "joinProject", (data = {}, callback) -> + WebsocketController.joinProject(client, user, data.project_id, callback) + \ No newline at end of file diff --git a/services/real-time/app/coffee/WebApiManager.coffee b/services/real-time/app/coffee/WebApiManager.coffee new file mode 100644 index 0000000000..f18f25c492 --- /dev/null +++ b/services/real-time/app/coffee/WebApiManager.coffee @@ -0,0 +1,21 @@ +request = require "request" +settings = require "settings-sharelatex" +logger = require "logger-sharelatex" + +module.exports = WebApiManager = + joinProject: (project_id, user_id, callback = (error, project, privilegeLevel) ->) -> + logger.log {project_id, user_id}, "sending join project request to web" + url = "#{settings.apis.web.url}/project/#{project_id}/join" + request.post { + url: url + qs: {user_id} + json: true + jar: false + }, (error, response, data) -> + return callback(error) if error? + if 200 <= response.statusCode < 300 + callback null, data?.project, data?.privilegeLevel + else + err = new Error("non-success status code from web: #{response.statusCode}") + logger.error {err, project_id, user_id}, "error accessing web api" + callback err \ No newline at end of file diff --git a/services/real-time/app/coffee/WebsocketController.coffee b/services/real-time/app/coffee/WebsocketController.coffee new file mode 100644 index 0000000000..bcf346d314 --- /dev/null +++ b/services/real-time/app/coffee/WebsocketController.coffee @@ -0,0 +1,31 @@ +logger = require "logger-sharelatex" +WebApiManager = require "./WebApiManager" + +module.exports = WebsocketController = + # If the protocol version changes when the client reconnects, + # it will force a full refresh of the page. Useful for non-backwards + # compatible protocol changes. Use only in extreme need. + PROTOCOL_VERSION: 2 + + joinProject: (client, user, project_id, callback = (error, project, privilegeLevel, protocolVersion) ->) -> + user_id = user?._id + logger.log {user_id, project_id}, "user joining project" + WebApiManager.joinProject project_id, user_id, (error, project, privilegeLevel) -> + return callback(error) if error? + + if !privilegeLevel or privilegeLevel == "" + err = new Error("not authorized") + logger.error {err, project_id, user_id}, "user is not authorized to join project" + return callback(err) + + client.set("user_id", user_id) + client.set("project_id", project_id) + client.set("owner_id", project?.owner?._id) + client.set("first_name", user?.first_name) + client.set("last_name", user?.last_name) + client.set("email", user?.email) + client.set("connected_time", new Date()) + client.set("signup_date", user?.signUpDate) + client.set("login_count", user?.loginCount) + + callback null, project, privilegeLevel, WebsocketController.PROTOCOL_VERSION \ No newline at end of file diff --git a/services/real-time/config/settings.defaults.coffee b/services/real-time/config/settings.defaults.coffee index 7b069975b1..f1b46c4638 100644 --- a/services/real-time/config/settings.defaults.coffee +++ b/services/real-time/config/settings.defaults.coffee @@ -10,6 +10,10 @@ module.exports = port: 3026 host: "localhost" + apis: + web: + url: "http://localhost:3000" + security: sessionSecret: "secret-please-change" diff --git a/services/real-time/package.json b/services/real-time/package.json index 01b664e945..5bc6c6b90e 100644 --- a/services/real-time/package.json +++ b/services/real-time/package.json @@ -34,6 +34,7 @@ "request": "~2.34.0", "sandboxed-module": "~0.3.0", "sinon": "~1.5.2", - "uid-safe": "^1.0.1" + "uid-safe": "^1.0.1", + "timekeeper": "0.0.4" } } diff --git a/services/real-time/test/acceptance/coffee/JoinProjectTests.coffee b/services/real-time/test/acceptance/coffee/JoinProjectTests.coffee new file mode 100644 index 0000000000..ef97f4783b --- /dev/null +++ b/services/real-time/test/acceptance/coffee/JoinProjectTests.coffee @@ -0,0 +1,44 @@ +chai = require("chai") +expect = chai.expect +chai.should() + +RealTimeClient = require "./helpers/RealTimeClient" +MockWebClient = require "./helpers/MockWebClient" + +describe "joinProject", -> + before (done) -> + @user_id = "mock-user-id" + @project_id = "mock-project-id" + privileges = {} + privileges[@user_id] = "owner" + MockWebClient.createMockProject(@project_id, privileges, { + name: "Test Project" + }) + MockWebClient.run (error) => + throw error if error? + RealTimeClient.setSession { + user: { _id: @user_id } + }, (error) => + throw error if error? + @client = RealTimeClient.connect() + @client.emit "joinProject", { + project_id: @project_id + }, (error, @project, @privilegeLevel, @protocolVersion) => + throw error if error? + done() + + it "should get the project from web", -> + MockWebClient.joinProject + .calledWith(@project_id, @user_id) + .should.equal true + + it "should return the project", -> + @project.should.deep.equal { + name: "Test Project" + } + + it "should return the privilege level", -> + @privilegeLevel.should.equal "owner" + + it "should return the protocolVersion", -> + @protocolVersion.should.equal 2 diff --git a/services/real-time/test/acceptance/coffee/SessionTests.coffee b/services/real-time/test/acceptance/coffee/SessionTests.coffee index 01b31cee01..2eb0d8206a 100644 --- a/services/real-time/test/acceptance/coffee/SessionTests.coffee +++ b/services/real-time/test/acceptance/coffee/SessionTests.coffee @@ -6,6 +6,7 @@ RealTimeClient = require "./helpers/RealTimeClient" describe "Session", -> describe "with an established session", -> beforeEach (done) -> + @user_id = "mock-user-id" RealTimeClient.setSession { user: { _id: @user_id } }, (error) => @@ -33,7 +34,7 @@ describe "Session", -> @client.on "disconnect", () -> done() - describe "with a user set on the session", -> + describe "without a valid user set on the session", -> beforeEach (done) -> RealTimeClient.setSession { foo: "bar" diff --git a/services/real-time/test/acceptance/coffee/helpers/MockWebClient.coffee b/services/real-time/test/acceptance/coffee/helpers/MockWebClient.coffee new file mode 100644 index 0000000000..b7c4ead1fc --- /dev/null +++ b/services/real-time/test/acceptance/coffee/helpers/MockWebClient.coffee @@ -0,0 +1,39 @@ +sinon = require "sinon" +express = require "express" + +module.exports = MockWebClient = + projects: {} + privileges: {} + + createMockProject: (project_id, privileges, project) -> + MockWebClient.privileges[project_id] = privileges + MockWebClient.projects[project_id] = project + + joinProject: (project_id, user_id, callback = (error, project, privilegeLevel) ->) -> + callback( + null, + MockWebClient.projects[project_id], + MockWebClient.privileges[project_id][user_id] + ) + + joinProjectRequest: (req, res, next) -> + {project_id} = req.params + {user_id} = req.query + MockWebClient.joinProject project_id, user_id, (error, project, privilegeLevel) -> + return next(error) if error? + res.json { + project: project + privilegeLevel: privilegeLevel + } + + running: false + run: (callback = (error) ->) -> + if MockWebClient.running + return callback() + app = express() + app.post "/project/:project_id/join", MockWebClient.joinProjectRequest + app.listen 3000, (error) -> + MockWebClient.running = true + callback(error) + +sinon.spy MockWebClient, "joinProject" \ No newline at end of file diff --git a/services/real-time/test/unit/coffee/WebApiManagerTests.coffee b/services/real-time/test/unit/coffee/WebApiManagerTests.coffee new file mode 100644 index 0000000000..4cc949dc9c --- /dev/null +++ b/services/real-time/test/unit/coffee/WebApiManagerTests.coffee @@ -0,0 +1,55 @@ +chai = require('chai') +should = chai.should() +sinon = require("sinon") +modulePath = "../../../app/js/WebApiManager.js" +SandboxedModule = require('sandboxed-module') + +describe 'WebApiManager', -> + beforeEach -> + @project_id = "project-id-123" + @user_id = "user-id-123" + @callback = sinon.stub() + @WebApiManager = SandboxedModule.require modulePath, requires: + "request": @request = {} + "settings-sharelatex": @settings = + apis: + web: + url: "http://web.example.com" + "logger-sharelatex": @logger = { log: sinon.stub(), error: sinon.stub() } + + describe "joinProject", -> + describe "successfully", -> + beforeEach -> + @response = { + project: { name: "Test project" } + privilegeLevel: "owner" + } + @request.post = sinon.stub().callsArgWith(1, null, {statusCode: 200}, @response) + @WebApiManager.joinProject @project_id, @user_id, @callback + + it "should send a request to web to join the project", -> + @request.post + .calledWith({ + url: "#{@settings.apis.web.url}/project/#{@project_id}/join" + qs: + user_id: @user_id + json: true + jar: false + }) + .should.equal true + + it "should return the project and privilegeLevel", -> + @callback + .calledWith(null, @response.project, @response.privilegeLevel) + .should.equal true + + describe "with an error from web", -> + beforeEach -> + @request.post = sinon.stub().callsArgWith(1, null, {statusCode: 500}, null) + @WebApiManager.joinProject @project_id, @user_id, @callback + + it "should call the callback with an error", -> + @callback + .calledWith(new Error("non-success code from web: 500")) + .should.equal true + \ No newline at end of file diff --git a/services/real-time/test/unit/coffee/WebsocketControllerTests.coffee b/services/real-time/test/unit/coffee/WebsocketControllerTests.coffee new file mode 100644 index 0000000000..be9ede3dad --- /dev/null +++ b/services/real-time/test/unit/coffee/WebsocketControllerTests.coffee @@ -0,0 +1,89 @@ +chai = require('chai') +should = chai.should() +sinon = require("sinon") +modulePath = "../../../app/js/WebsocketController.js" +SandboxedModule = require('sandboxed-module') +tk = require "timekeeper" + +describe 'WebsocketController', -> + beforeEach -> + tk.freeze(new Date()) + @project_id = "project-id-123" + @user = { + _id: "user-id-123" + first_name: "James" + last_name: "Allen" + email: "james@example.com" + signUpDate: new Date("2014-01-01") + loginCount: 42 + } + @callback = sinon.stub() + @client = + set: sinon.stub() + join: sinon.stub() + @WebsocketController = SandboxedModule.require modulePath, requires: + "./WebApiManager": @WebApiManager = {} + "logger-sharelatex": @logger = { log: sinon.stub(), error: sinon.stub() } + + afterEach -> + tk.reset() + + describe "joinProject", -> + describe "when authorised", -> + beforeEach -> + @project = { + name: "Test Project" + owner: { + _id: @owner_id = "mock-owner-id-123" + } + } + @privilegeLevel = "owner" + @WebApiManager.joinProject = sinon.stub().callsArgWith(2, null, @project, @privilegeLevel) + @WebsocketController.joinProject @client, @user, @project_id, @callback + + it "should load the project from web", -> + @WebApiManager.joinProject + .calledWith(@project_id, @user._id) + .should.equal true + + it "should set the user's id on the client", -> + @client.set.calledWith("user_id", @user._id).should.equal true + + it "should set the user's email on the client", -> + @client.set.calledWith("email", @user.email).should.equal true + + it "should set the user's first_name on the client", -> + @client.set.calledWith("first_name", @user.first_name).should.equal true + + it "should set the user's last_name on the client", -> + @client.set.calledWith("last_name", @user.last_name).should.equal true + + it "should set the user's sign up date on the client", -> + @client.set.calledWith("signup_date", @user.signUpDate).should.equal true + + it "should set the user's login_count on the client", -> + @client.set.calledWith("login_count", @user.loginCount).should.equal true + + it "should set the connected time on the client", -> + @client.set.calledWith("connected_time", new Date()).should.equal true + + it "should set the project_id on the client", -> + @client.set.calledWith("project_id", @project_id).should.equal true + + it "should set the project owner id on the client", -> + @client.set.calledWith("owner_id", @owner_id).should.equal true + + it "should call the callback with the project, privilegeLevel and protocolVersion", -> + @callback + .calledWith(null, @project, @privilegeLevel, @WebsocketController.PROTOCOL_VERSION) + .should.equal true + + describe "when not authorized", -> + beforeEach -> + @WebApiManager.joinProject = sinon.stub().callsArgWith(2, null, null, null) + @WebsocketController.joinProject @client, @user, @project_id, @callback + + it "should return an error", -> + @callback + .calledWith(new Error("not authorized")) + .should.equal true \ No newline at end of file From dc60f2b7362ac09196f88e759fdf2028db405910 Mon Sep 17 00:00:00 2001 From: James Allen Date: Mon, 10 Nov 2014 11:38:26 +0000 Subject: [PATCH 003/491] Add acceptance test for unauthorized project joining --- .../app/coffee/WebsocketController.coffee | 4 +- .../acceptance/coffee/JoinProjectTests.coffee | 92 ++++++++++++------- .../coffee/WebsocketControllerTests.coffee | 2 +- 3 files changed, 62 insertions(+), 36 deletions(-) diff --git a/services/real-time/app/coffee/WebsocketController.coffee b/services/real-time/app/coffee/WebsocketController.coffee index bcf346d314..41ce2d7464 100644 --- a/services/real-time/app/coffee/WebsocketController.coffee +++ b/services/real-time/app/coffee/WebsocketController.coffee @@ -16,7 +16,9 @@ module.exports = WebsocketController = if !privilegeLevel or privilegeLevel == "" err = new Error("not authorized") logger.error {err, project_id, user_id}, "user is not authorized to join project" - return callback(err) + # Don't send an error object since socket.io can apparently + # only serialize JSON. + return callback({message: err.message}) client.set("user_id", user_id) client.set("project_id", project_id) diff --git a/services/real-time/test/acceptance/coffee/JoinProjectTests.coffee b/services/real-time/test/acceptance/coffee/JoinProjectTests.coffee index ef97f4783b..84f61c6476 100644 --- a/services/real-time/test/acceptance/coffee/JoinProjectTests.coffee +++ b/services/real-time/test/acceptance/coffee/JoinProjectTests.coffee @@ -6,39 +6,63 @@ RealTimeClient = require "./helpers/RealTimeClient" MockWebClient = require "./helpers/MockWebClient" describe "joinProject", -> - before (done) -> - @user_id = "mock-user-id" - @project_id = "mock-project-id" - privileges = {} - privileges[@user_id] = "owner" - MockWebClient.createMockProject(@project_id, privileges, { - name: "Test Project" - }) - MockWebClient.run (error) => - throw error if error? - RealTimeClient.setSession { - user: { _id: @user_id } - }, (error) => + describe "when authorized", -> + before (done) -> + @user_id = "mock-user-id" + @project_id = "mock-project-id" + privileges = {} + privileges[@user_id] = "owner" + MockWebClient.createMockProject(@project_id, privileges, { + name: "Test Project" + }) + MockWebClient.run (error) => throw error if error? - @client = RealTimeClient.connect() - @client.emit "joinProject", { - project_id: @project_id - }, (error, @project, @privilegeLevel, @protocolVersion) => + RealTimeClient.setSession { + user: { _id: @user_id } + }, (error) => throw error if error? - done() - - it "should get the project from web", -> - MockWebClient.joinProject - .calledWith(@project_id, @user_id) - .should.equal true - - it "should return the project", -> - @project.should.deep.equal { - name: "Test Project" - } - - it "should return the privilege level", -> - @privilegeLevel.should.equal "owner" - - it "should return the protocolVersion", -> - @protocolVersion.should.equal 2 + @client = RealTimeClient.connect() + @client.emit "joinProject", { + project_id: @project_id + }, (error, @project, @privilegeLevel, @protocolVersion) => + throw error if error? + done() + + it "should get the project from web", -> + MockWebClient.joinProject + .calledWith(@project_id, @user_id) + .should.equal true + + it "should return the project", -> + @project.should.deep.equal { + name: "Test Project" + } + + it "should return the privilege level", -> + @privilegeLevel.should.equal "owner" + + it "should return the protocolVersion", -> + @protocolVersion.should.equal 2 + + describe "when not authorized", -> + before (done) -> + @user_id = "mock-user-id-2" + @project_id = "mock-project-id-2" + privileges = {} + MockWebClient.createMockProject(@project_id, privileges, { + name: "Test Project" + }) + MockWebClient.run (error) => + throw error if error? + RealTimeClient.setSession { + user: { _id: @user_id } + }, (error) => + throw error if error? + @client = RealTimeClient.connect() + @client.emit "joinProject", { + project_id: @project_id + }, (@error, @project, @privilegeLevel, @protocolVersion) => + done() + + it "should return an error", -> + @error.message.should.equal "not authorized" diff --git a/services/real-time/test/unit/coffee/WebsocketControllerTests.coffee b/services/real-time/test/unit/coffee/WebsocketControllerTests.coffee index be9ede3dad..cec6c0fc6b 100644 --- a/services/real-time/test/unit/coffee/WebsocketControllerTests.coffee +++ b/services/real-time/test/unit/coffee/WebsocketControllerTests.coffee @@ -85,5 +85,5 @@ describe 'WebsocketController', -> it "should return an error", -> @callback - .calledWith(new Error("not authorized")) + .calledWith({message: "not authorized"}) .should.equal true \ No newline at end of file From 919b192e16eda0d717deae62cf5d54ec26eaffa1 Mon Sep 17 00:00:00 2001 From: James Allen Date: Mon, 10 Nov 2014 11:40:19 +0000 Subject: [PATCH 004/491] Add in null check --- services/real-time/app/coffee/Router.coffee | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/services/real-time/app/coffee/Router.coffee b/services/real-time/app/coffee/Router.coffee index 73c3aa0acd..23108921b7 100644 --- a/services/real-time/app/coffee/Router.coffee +++ b/services/real-time/app/coffee/Router.coffee @@ -21,5 +21,5 @@ module.exports = Router = return client.on "joinProject", (data = {}, callback) -> - WebsocketController.joinProject(client, user, data.project_id, callback) + WebsocketController.joinProject(client, user, data?.project_id, callback) \ No newline at end of file From eb8ccc0298f45f9063cdbbf91fda18ee33209be6 Mon Sep 17 00:00:00 2001 From: James Allen Date: Wed, 12 Nov 2014 15:54:55 +0000 Subject: [PATCH 005/491] Create joinDoc socket.io end point --- .../app/coffee/AuthorizationManager.coffee | 12 ++ .../app/coffee/DocumentUpdaterManager.coffee | 26 ++++ services/real-time/app/coffee/Router.coffee | 44 ++++++- .../app/coffee/WebsocketController.coffee | 39 +++++- .../real-time/config/settings.defaults.coffee | 2 + .../acceptance/coffee/JoinDocTests.coffee | 118 ++++++++++++++++++ .../acceptance/coffee/JoinProjectTests.coffee | 66 +++++----- .../coffee/helpers/FixturesManager.coffee | 42 +++++++ .../helpers/MockDocUpdaterServer.coffee | 33 +++++ ...kWebClient.coffee => MockWebServer.coffee} | 20 +-- .../coffee/AuthorizationManagerTests.coffee | 39 ++++++ .../coffee/DocumentUpdaterManagerTests.coffee | 60 +++++++++ .../coffee/WebsocketControllerTests.coffee | 71 ++++++++++- 13 files changed, 516 insertions(+), 56 deletions(-) create mode 100644 services/real-time/app/coffee/AuthorizationManager.coffee create mode 100644 services/real-time/app/coffee/DocumentUpdaterManager.coffee create mode 100644 services/real-time/test/acceptance/coffee/JoinDocTests.coffee create mode 100644 services/real-time/test/acceptance/coffee/helpers/FixturesManager.coffee create mode 100644 services/real-time/test/acceptance/coffee/helpers/MockDocUpdaterServer.coffee rename services/real-time/test/acceptance/coffee/helpers/{MockWebClient.coffee => MockWebServer.coffee} (58%) create mode 100644 services/real-time/test/unit/coffee/AuthorizationManagerTests.coffee create mode 100644 services/real-time/test/unit/coffee/DocumentUpdaterManagerTests.coffee diff --git a/services/real-time/app/coffee/AuthorizationManager.coffee b/services/real-time/app/coffee/AuthorizationManager.coffee new file mode 100644 index 0000000000..345bfa4804 --- /dev/null +++ b/services/real-time/app/coffee/AuthorizationManager.coffee @@ -0,0 +1,12 @@ +module.exports = AuthorizationManager = + assertClientCanViewProject: (client, callback = (error) ->) -> + AuthorizationManager._assertClientHasPrivilegeLevel client, ["readOnly", "readAndWrite", "owner"], callback + + _assertClientHasPrivilegeLevel: (client, allowedLevels, callback = (error) ->) -> + client.get "privilege_level", (error, privilegeLevel) -> + return callback(error) if error? + allowed = (privilegeLevel in allowedLevels) + if allowed + callback null + else + callback new Error("not authorized") \ No newline at end of file diff --git a/services/real-time/app/coffee/DocumentUpdaterManager.coffee b/services/real-time/app/coffee/DocumentUpdaterManager.coffee new file mode 100644 index 0000000000..54ba6f9072 --- /dev/null +++ b/services/real-time/app/coffee/DocumentUpdaterManager.coffee @@ -0,0 +1,26 @@ +request = require "request" +logger = require "logger-sharelatex" +settings = require "settings-sharelatex" + +module.exports = DocumentUpdaterManager = + getDocument: (project_id, doc_id, fromVersion, callback = (error, exists, doclines, version) ->) -> + #timer = new metrics.Timer("get-document") + url = "#{settings.apis.documentupdater.url}/project/#{project_id}/doc/#{doc_id}?fromVersion=#{fromVersion}" + logger.log {project_id, doc_id, fromVersion}, "getting doc from document updater" + request.get url, (err, res, body) -> + #timer.done() + if err? + logger.error {err, url, project_id, doc_id}, "error getting doc from doc updater" + return callback(err) + if 200 <= res.statusCode < 300 + logger.log {project_id, doc_id}, "got doc from document document updater" + try + body = JSON.parse(body) + catch error + return callback(error) + callback null, body?.lines, body?.version, body?.ops + else + err = new Error("doc updater returned a non-success status code: #{res.statusCode}") + err.statusCode = res.statusCode + logger.error {err, project_id, doc_id, url}, "doc updater returned a non-success status code: #{res.statusCode}" + callback err \ No newline at end of file diff --git a/services/real-time/app/coffee/Router.coffee b/services/real-time/app/coffee/Router.coffee index 23108921b7..06cd7bb7a0 100644 --- a/services/real-time/app/coffee/Router.coffee +++ b/services/real-time/app/coffee/Router.coffee @@ -3,6 +3,23 @@ logger = require "logger-sharelatex" WebsocketController = require "./WebsocketController" module.exports = Router = + # We don't want to send raw errors back to the client, in case they + # contain sensitive data. Instead we log them out, and send a generic + # JSON object which can be serialized over socket.io + _createCallbackWithErrorFilter: (client, method, callback) -> + return (err, args...) -> + if err? + + err = {message: "Something went wrong"} + callback err, args... + + # Used in error reporting + _getClientData: (client, callback = (error, data) ->) -> + client.get "user_id", (error, user_id) -> + client.get "project_id", (error, project_id) -> + client.get "doc_id", (error, doc_id) -> + callback null, { id: client.id, user_id, project_id, doc_id } + configure: (app, io, session) -> session.on 'connection', (error, client, session) -> if error? @@ -12,7 +29,7 @@ module.exports = Router = Metrics.inc('socket-io.connection') - logger.log session: session, "got session" + logger.log session: session, client_id: client.id, "client connected" user = session.user if !user? or !user._id? @@ -21,5 +38,28 @@ module.exports = Router = return client.on "joinProject", (data = {}, callback) -> - WebsocketController.joinProject(client, user, data?.project_id, callback) + WebsocketController.joinProject client, user, data.project_id, (err, args...) -> + if err? + Router._getClientData client, (_, client) -> + logger.error {err, client, project_id: data.project_id}, "server side error in joinProject" + # Don't return raw error to prevent leaking server side info + return callback {message: "Something went wrong"} + else + callback(null, args...) + + + client.on "joinDoc", (doc_id, fromVersion, callback) -> + # fromVersion is optional + if typeof fromVersion == "function" + callback = fromVersion + fromVersion = -1 + + WebsocketController.joinDoc client, doc_id, fromVersion, (err, args...) -> + if err? + Router._getClientData client, (_, client) -> + logger.error {err, client, doc_id, fromVersion}, "server side error in joinDoc" + # Don't return raw error to prevent leaking server side info + return callback {message: "Something went wrong"} + else + callback(null, args...) \ No newline at end of file diff --git a/services/real-time/app/coffee/WebsocketController.coffee b/services/real-time/app/coffee/WebsocketController.coffee index 41ce2d7464..f409f39375 100644 --- a/services/real-time/app/coffee/WebsocketController.coffee +++ b/services/real-time/app/coffee/WebsocketController.coffee @@ -1,5 +1,7 @@ logger = require "logger-sharelatex" WebApiManager = require "./WebApiManager" +AuthorizationManager = require "./AuthorizationManager" +DocumentUpdaterManager = require "./DocumentUpdaterManager" module.exports = WebsocketController = # If the protocol version changes when the client reconnects, @@ -9,17 +11,16 @@ module.exports = WebsocketController = joinProject: (client, user, project_id, callback = (error, project, privilegeLevel, protocolVersion) ->) -> user_id = user?._id - logger.log {user_id, project_id}, "user joining project" + logger.log {user_id, project_id, client_id: client.id}, "user joining project" WebApiManager.joinProject project_id, user_id, (error, project, privilegeLevel) -> return callback(error) if error? if !privilegeLevel or privilegeLevel == "" err = new Error("not authorized") - logger.error {err, project_id, user_id}, "user is not authorized to join project" - # Don't send an error object since socket.io can apparently - # only serialize JSON. - return callback({message: err.message}) + logger.error {err, project_id, user_id, client_id: client.id}, "user is not authorized to join project" + return callback(err) + client.set("privilege_level", privilegeLevel) client.set("user_id", user_id) client.set("project_id", project_id) client.set("owner_id", project?.owner?._id) @@ -30,4 +31,30 @@ module.exports = WebsocketController = client.set("signup_date", user?.signUpDate) client.set("login_count", user?.loginCount) - callback null, project, privilegeLevel, WebsocketController.PROTOCOL_VERSION \ No newline at end of file + callback null, project, privilegeLevel, WebsocketController.PROTOCOL_VERSION + + joinDoc: (client, doc_id, fromVersion = -1, callback = (error, doclines, version, ops) ->) -> + client.get "user_id", (error, user_id) -> + client.get "project_id", (error, project_id) -> + logger.log {user_id, project_id, doc_id, fromVersion, client_id: client.id}, "client joining doc" + + AuthorizationManager.assertClientCanViewProject client, (error) -> + return callback(error) if error? + client.get "project_id", (error, project_id) -> + return callback(error) if error? + return callback(new Error("no project_id found on client")) if !project_id? + DocumentUpdaterManager.getDocument project_id, doc_id, fromVersion, (error, lines, version, ops) -> + return callback(error) if error? + # Encode any binary bits of data so it can go via WebSockets + # See http://ecmanaut.blogspot.co.uk/2006/07/encoding-decoding-utf8-in-javascript.html + escapedLines = [] + for line in lines + try + line = unescape(encodeURIComponent(line)) + catch err + logger.err {err, project_id, doc_id, fromVersion, line, client_id: client.id}, "error encoding line uri component" + return callback(err) + escapedLines.push line + client.join(doc_id) + callback null, escapedLines, version, ops + \ No newline at end of file diff --git a/services/real-time/config/settings.defaults.coffee b/services/real-time/config/settings.defaults.coffee index f1b46c4638..7d9ddf184c 100644 --- a/services/real-time/config/settings.defaults.coffee +++ b/services/real-time/config/settings.defaults.coffee @@ -13,6 +13,8 @@ module.exports = apis: web: url: "http://localhost:3000" + documentupdater: + url: "http://localhost:3003" security: sessionSecret: "secret-please-change" diff --git a/services/real-time/test/acceptance/coffee/JoinDocTests.coffee b/services/real-time/test/acceptance/coffee/JoinDocTests.coffee new file mode 100644 index 0000000000..a339cf7ef8 --- /dev/null +++ b/services/real-time/test/acceptance/coffee/JoinDocTests.coffee @@ -0,0 +1,118 @@ +chai = require("chai") +expect = chai.expect +chai.should() + +RealTimeClient = require "./helpers/RealTimeClient" +MockDocUpdaterServer = require "./helpers/MockDocUpdaterServer" +FixturesManager = require "./helpers/FixturesManager" + +describe "joinDoc", -> + before -> + @lines = ["test", "doc", "lines"] + @version = 42 + @ops = ["mock", "doc", "ops"] + + describe "when authorised readAndWrite", -> + before (done) -> + FixturesManager.setUpProject { + privilegeLevel: "readAndWrite" + }, (error, data) => + throw error if error? + {@project_id, @user_id} = data + FixturesManager.setUpDoc @project_id, {@lines, @version, @ops}, (error, data) => + throw error if error? + {@doc_id} = data + @client = RealTimeClient.connect() + @client.emit "joinProject", project_id: @project_id, (error) => + throw error if error? + @client.emit "joinDoc", @doc_id, (error, @returnedArgs...) => + throw error if error? + done() + + it "should get the doc from the doc updater", -> + MockDocUpdaterServer.getDocument + .calledWith(@project_id, @doc_id, -1) + .should.equal true + + it "should return the doc lines, version and ops", -> + @returnedArgs.should.deep.equal [@lines, @version, @ops] + + describe "when authorised readOnly", -> + before (done) -> + FixturesManager.setUpProject { + privilegeLevel: "readOnly" + }, (error, data) => + throw error if error? + {@project_id, @user_id} = data + FixturesManager.setUpDoc @project_id, {@lines, @version, @ops}, (error, data) => + throw error if error? + {@doc_id} = data + @client = RealTimeClient.connect() + @client.emit "joinProject", project_id: @project_id, (error) => + throw error if error? + @client.emit "joinDoc", @doc_id, (error, @returnedArgs...) => + throw error if error? + done() + + it "should get the doc from the doc updater", -> + MockDocUpdaterServer.getDocument + .calledWith(@project_id, @doc_id, -1) + .should.equal true + + it "should return the doc lines, version and ops", -> + @returnedArgs.should.deep.equal [@lines, @version, @ops] + + describe "when authorised as owner", -> + before (done) -> + FixturesManager.setUpProject { + privilegeLevel: "owner" + }, (error, data) => + throw error if error? + {@project_id, @user_id} = data + FixturesManager.setUpDoc @project_id, {@lines, @version, @ops}, (error, data) => + throw error if error? + {@doc_id} = data + @client = RealTimeClient.connect() + @client.emit "joinProject", project_id: @project_id, (error) => + throw error if error? + @client.emit "joinDoc", @doc_id, (error, @returnedArgs...) => + throw error if error? + done() + + it "should get the doc from the doc updater", -> + MockDocUpdaterServer.getDocument + .calledWith(@project_id, @doc_id, -1) + .should.equal true + + it "should return the doc lines, version and ops", -> + @returnedArgs.should.deep.equal [@lines, @version, @ops] + + # It is impossible to write an acceptance test to test joining an unauthorized + # project, since joinProject already catches that. If you can join a project, + # then you can join a doc in that project. + + describe "with a fromVersion", -> + before (done) -> + @fromVersion = 36 + FixturesManager.setUpProject { + privilegeLevel: "readAndWrite" + }, (error, data) => + throw error if error? + {@project_id, @user_id} = data + FixturesManager.setUpDoc @project_id, {@lines, @version, @ops}, (error, data) => + throw error if error? + {@doc_id} = data + @client = RealTimeClient.connect() + @client.emit "joinProject", project_id: @project_id, (error) => + throw error if error? + @client.emit "joinDoc", @doc_id, @fromVersion, (error, @returnedArgs...) => + throw error if error? + done() + + it "should get the doc from the doc updater with the fromVersion", -> + MockDocUpdaterServer.getDocument + .calledWith(@project_id, @doc_id, @fromVersion) + .should.equal true + + it "should return the doc lines, version and ops", -> + @returnedArgs.should.deep.equal [@lines, @version, @ops] \ No newline at end of file diff --git a/services/real-time/test/acceptance/coffee/JoinProjectTests.coffee b/services/real-time/test/acceptance/coffee/JoinProjectTests.coffee index 84f61c6476..02be863b9c 100644 --- a/services/real-time/test/acceptance/coffee/JoinProjectTests.coffee +++ b/services/real-time/test/acceptance/coffee/JoinProjectTests.coffee @@ -3,33 +3,30 @@ expect = chai.expect chai.should() RealTimeClient = require "./helpers/RealTimeClient" -MockWebClient = require "./helpers/MockWebClient" +MockWebServer = require "./helpers/MockWebServer" +FixturesManager = require "./helpers/FixturesManager" + describe "joinProject", -> describe "when authorized", -> before (done) -> - @user_id = "mock-user-id" - @project_id = "mock-project-id" - privileges = {} - privileges[@user_id] = "owner" - MockWebClient.createMockProject(@project_id, privileges, { - name: "Test Project" - }) - MockWebClient.run (error) => + FixturesManager.setUpProject { + privilegeLevel: "owner" + project: { + name: "Test Project" + } + }, (error, data) => throw error if error? - RealTimeClient.setSession { - user: { _id: @user_id } - }, (error) => + {@user_id, @project_id} = data + @client = RealTimeClient.connect() + @client.emit "joinProject", { + project_id: @project_id + }, (error, @project, @privilegeLevel, @protocolVersion) => throw error if error? - @client = RealTimeClient.connect() - @client.emit "joinProject", { - project_id: @project_id - }, (error, @project, @privilegeLevel, @protocolVersion) => - throw error if error? - done() + done() it "should get the project from web", -> - MockWebClient.joinProject + MockWebServer.joinProject .calledWith(@project_id, @user_id) .should.equal true @@ -46,23 +43,20 @@ describe "joinProject", -> describe "when not authorized", -> before (done) -> - @user_id = "mock-user-id-2" - @project_id = "mock-project-id-2" - privileges = {} - MockWebClient.createMockProject(@project_id, privileges, { - name: "Test Project" - }) - MockWebClient.run (error) => + FixturesManager.setUpProject { + privilegeLevel: null + project: { + name: "Test Project" + } + }, (error, data) => throw error if error? - RealTimeClient.setSession { - user: { _id: @user_id } - }, (error) => - throw error if error? - @client = RealTimeClient.connect() - @client.emit "joinProject", { - project_id: @project_id - }, (@error, @project, @privilegeLevel, @protocolVersion) => - done() + {@user_id, @project_id} = data + @client = RealTimeClient.connect() + @client.emit "joinProject", { + project_id: @project_id + }, (@error, @project, @privilegeLevel, @protocolVersion) => + done() it "should return an error", -> - @error.message.should.equal "not authorized" + # We don't return specific errors + @error.message.should.equal "Something went wrong" diff --git a/services/real-time/test/acceptance/coffee/helpers/FixturesManager.coffee b/services/real-time/test/acceptance/coffee/helpers/FixturesManager.coffee new file mode 100644 index 0000000000..5c2eb5fd3a --- /dev/null +++ b/services/real-time/test/acceptance/coffee/helpers/FixturesManager.coffee @@ -0,0 +1,42 @@ +RealTimeClient = require "./RealTimeClient" +MockWebServer = require "./MockWebServer" +MockDocUpdaterServer = require "./MockDocUpdaterServer" + +module.exports = FixturesManager = + setUpProject: (options = {}, callback = (error, data) ->) -> + options.user_id ||= FixturesManager.getRandomId() + options.project_id ||= FixturesManager.getRandomId() + options.project ||= { name: "Test Project" } + {project_id, user_id, privilegeLevel, project} = options + + privileges = {} + privileges[user_id] = privilegeLevel + + MockWebServer.createMockProject(project_id, privileges, project) + MockWebServer.run (error) => + throw error if error? + RealTimeClient.setSession { + user: { _id: user_id } + }, (error) => + throw error if error? + callback null, {project_id, user_id, privilegeLevel, project} + + setUpDoc: (project_id, options = {}, callback = (error, data) ->) -> + options.doc_id ||= FixturesManager.getRandomId() + options.lines ||= ["doc", "lines"] + options.version ||= 42 + options.ops ||= ["mock", "ops"] + {doc_id, lines, version, ops} = options + + MockDocUpdaterServer.createMockDoc project_id, doc_id, {lines, version, ops} + MockDocUpdaterServer.run (error) => + throw error if error? + callback null, {project_id, doc_id, lines, version, ops} + + getRandomId: () -> + return require("crypto") + .createHash("sha1") + .update(Math.random().toString()) + .digest("hex") + .slice(0,24) + \ No newline at end of file diff --git a/services/real-time/test/acceptance/coffee/helpers/MockDocUpdaterServer.coffee b/services/real-time/test/acceptance/coffee/helpers/MockDocUpdaterServer.coffee new file mode 100644 index 0000000000..094d10aceb --- /dev/null +++ b/services/real-time/test/acceptance/coffee/helpers/MockDocUpdaterServer.coffee @@ -0,0 +1,33 @@ +sinon = require "sinon" +express = require "express" + +module.exports = MockDocUpdaterServer = + docs: {} + + createMockDoc: (project_id, doc_id, data) -> + MockDocUpdaterServer.docs["#{project_id}:#{doc_id}"] = data + + getDocument: (project_id, doc_id, fromVersion, callback = (error, data) ->) -> + callback( + null, MockDocUpdaterServer.docs["#{project_id}:#{doc_id}"] + ) + + getDocumentRequest: (req, res, next) -> + {project_id, doc_id} = req.params + {fromVersion} = req.query + fromVersion = parseInt(fromVersion, 10) + MockDocUpdaterServer.getDocument project_id, doc_id, fromVersion, (error, data) -> + return next(error) if error? + res.json data + + running: false + run: (callback = (error) ->) -> + if MockDocUpdaterServer.running + return callback() + app = express() + app.get "/project/:project_id/doc/:doc_id", MockDocUpdaterServer.getDocumentRequest + app.listen 3003, (error) -> + MockDocUpdaterServer.running = true + callback(error) + +sinon.spy MockDocUpdaterServer, "getDocument" \ No newline at end of file diff --git a/services/real-time/test/acceptance/coffee/helpers/MockWebClient.coffee b/services/real-time/test/acceptance/coffee/helpers/MockWebServer.coffee similarity index 58% rename from services/real-time/test/acceptance/coffee/helpers/MockWebClient.coffee rename to services/real-time/test/acceptance/coffee/helpers/MockWebServer.coffee index b7c4ead1fc..2fff23e252 100644 --- a/services/real-time/test/acceptance/coffee/helpers/MockWebClient.coffee +++ b/services/real-time/test/acceptance/coffee/helpers/MockWebServer.coffee @@ -1,25 +1,25 @@ sinon = require "sinon" express = require "express" -module.exports = MockWebClient = +module.exports = MockWebServer = projects: {} privileges: {} createMockProject: (project_id, privileges, project) -> - MockWebClient.privileges[project_id] = privileges - MockWebClient.projects[project_id] = project + MockWebServer.privileges[project_id] = privileges + MockWebServer.projects[project_id] = project joinProject: (project_id, user_id, callback = (error, project, privilegeLevel) ->) -> callback( null, - MockWebClient.projects[project_id], - MockWebClient.privileges[project_id][user_id] + MockWebServer.projects[project_id], + MockWebServer.privileges[project_id][user_id] ) joinProjectRequest: (req, res, next) -> {project_id} = req.params {user_id} = req.query - MockWebClient.joinProject project_id, user_id, (error, project, privilegeLevel) -> + MockWebServer.joinProject project_id, user_id, (error, project, privilegeLevel) -> return next(error) if error? res.json { project: project @@ -28,12 +28,12 @@ module.exports = MockWebClient = running: false run: (callback = (error) ->) -> - if MockWebClient.running + if MockWebServer.running return callback() app = express() - app.post "/project/:project_id/join", MockWebClient.joinProjectRequest + app.post "/project/:project_id/join", MockWebServer.joinProjectRequest app.listen 3000, (error) -> - MockWebClient.running = true + MockWebServer.running = true callback(error) -sinon.spy MockWebClient, "joinProject" \ No newline at end of file +sinon.spy MockWebServer, "joinProject" \ No newline at end of file diff --git a/services/real-time/test/unit/coffee/AuthorizationManagerTests.coffee b/services/real-time/test/unit/coffee/AuthorizationManagerTests.coffee new file mode 100644 index 0000000000..cd3822538b --- /dev/null +++ b/services/real-time/test/unit/coffee/AuthorizationManagerTests.coffee @@ -0,0 +1,39 @@ +chai = require "chai" +chai.should() +expect = chai.expect +sinon = require("sinon") +SandboxedModule = require('sandboxed-module') +path = require "path" +modulePath = '../../../app/js/AuthorizationManager' + +describe 'AuthorizationManager', -> + beforeEach -> + @client = + params: {} + get: (param, cb) -> cb null, @params[param] + @AuthorizationManager = SandboxedModule.require modulePath, requires: {} + + describe "assertClientCanViewProject", -> + it "should allow the readOnly privilegeLevel", (done) -> + @client.params.privilege_level = "readOnly" + @AuthorizationManager.assertClientCanViewProject @client, (error) -> + expect(error).to.be.null + done() + + it "should allow the readAndWrite privilegeLevel", (done) -> + @client.params.privilege_level = "readAndWrite" + @AuthorizationManager.assertClientCanViewProject @client, (error) -> + expect(error).to.be.null + done() + + it "should allow the owner privilegeLevel", (done) -> + @client.params.privilege_level = "owner" + @AuthorizationManager.assertClientCanViewProject @client, (error) -> + expect(error).to.be.null + done() + + it "should return an error with any other privilegeLevel", (done) -> + @client.params.privilege_level = "unknown" + @AuthorizationManager.assertClientCanViewProject @client, (error) -> + error.message.should.equal "not authorized" + done() \ No newline at end of file diff --git a/services/real-time/test/unit/coffee/DocumentUpdaterManagerTests.coffee b/services/real-time/test/unit/coffee/DocumentUpdaterManagerTests.coffee new file mode 100644 index 0000000000..9f80981374 --- /dev/null +++ b/services/real-time/test/unit/coffee/DocumentUpdaterManagerTests.coffee @@ -0,0 +1,60 @@ +require('chai').should() +sinon = require("sinon") +SandboxedModule = require('sandboxed-module') +path = require "path" +modulePath = '../../../app/js/DocumentUpdaterManager' + +describe 'DocumentUpdaterManager', -> + beforeEach -> + @project_id = "project-id-923" + @doc_id = "doc-id-394" + @lines = ["one", "two", "three"] + @version = 42 + @settings = + apis: documentupdater: url: "http://doc-updater.example.com" + + @DocumentUpdaterManager = SandboxedModule.require modulePath, requires: + 'settings-sharelatex':@settings + 'logger-sharelatex': @logger = {log: sinon.stub(), error: sinon.stub()} + 'request': @request = {} + + describe "getDocument", -> + beforeEach -> + @callback = sinon.stub() + + describe "successfully", -> + beforeEach -> + @body = JSON.stringify + lines: @lines + version: @version + ops: @ops = ["mock-op-1", "mock-op-2"] + @fromVersion = 2 + @request.get = sinon.stub().callsArgWith(1, null, {statusCode: 200}, @body) + @DocumentUpdaterManager.getDocument @project_id, @doc_id, @fromVersion, @callback + + it 'should get the document from the document updater', -> + url = "#{@settings.apis.documentupdater.url}/project/#{@project_id}/doc/#{@doc_id}?fromVersion=#{@fromVersion}" + @request.get.calledWith(url).should.equal true + + it "should call the callback with the lines and version", -> + @callback.calledWith(null, @lines, @version, @ops).should.equal true + + describe "when the document updater API returns an error", -> + beforeEach -> + @request.get = sinon.stub().callsArgWith(1, @error = new Error("something went wrong"), null, null) + @DocumentUpdaterManager.getDocument @project_id, @doc_id, @fromVersion, @callback + + it "should return an error to the callback", -> + @callback.calledWith(@error).should.equal true + + describe "when the document updater returns a failure error code", -> + beforeEach -> + @request.get = sinon.stub().callsArgWith(1, null, { statusCode: 500 }, "") + @DocumentUpdaterManager.getDocument @project_id, @doc_id, @fromVersion, @callback + + it "should return the callback with an error", -> + err = new Error("doc updater returned failure status code: 500") + err.statusCode = 500 + @callback + .calledWith(err) + .should.equal true diff --git a/services/real-time/test/unit/coffee/WebsocketControllerTests.coffee b/services/real-time/test/unit/coffee/WebsocketControllerTests.coffee index cec6c0fc6b..eea0e7e1f0 100644 --- a/services/real-time/test/unit/coffee/WebsocketControllerTests.coffee +++ b/services/real-time/test/unit/coffee/WebsocketControllerTests.coffee @@ -19,10 +19,14 @@ describe 'WebsocketController', -> } @callback = sinon.stub() @client = + params: {} set: sinon.stub() + get: (param, cb) -> cb null, @params[param] join: sinon.stub() @WebsocketController = SandboxedModule.require modulePath, requires: "./WebApiManager": @WebApiManager = {} + "./AuthorizationManager": @AuthorizationManager = {} + "./DocumentUpdaterManager": @DocumentUpdaterManager = {} "logger-sharelatex": @logger = { log: sinon.stub(), error: sinon.stub() } afterEach -> @@ -46,6 +50,9 @@ describe 'WebsocketController', -> .calledWith(@project_id, @user._id) .should.equal true + it "should set the privilege level on the client", -> + @client.set.calledWith("privilege_level", @privilegeLevel).should.equal true + it "should set the user's id on the client", -> @client.set.calledWith("user_id", @user._id).should.equal true @@ -85,5 +92,65 @@ describe 'WebsocketController', -> it "should return an error", -> @callback - .calledWith({message: "not authorized"}) - .should.equal true \ No newline at end of file + .calledWith(new Error("not authorized")) + .should.equal true + + describe "joinDoc", -> + beforeEach -> + @doc_id = "doc-id-123" + @doc_lines = ["doc", "lines"] + @version = 42 + @ops = ["mock", "ops"] + + @client.params.project_id = @project_id + + @AuthorizationManager.assertClientCanViewProject = sinon.stub().callsArgWith(1, null) + @DocumentUpdaterManager.getDocument = sinon.stub().callsArgWith(3, null, @doc_lines, @version, @ops) + + describe "with a fromVersion", -> + beforeEach -> + @fromVersion = 40 + @WebsocketController.joinDoc @client, @doc_id, @fromVersion, @callback + + it "should check that the client is authorized to view the project", -> + @AuthorizationManager.assertClientCanViewProject + .calledWith(@client) + .should.equal true + + it "should get the document from the DocumentUpdaterManager", -> + @DocumentUpdaterManager.getDocument + .calledWith(@project_id, @doc_id, @fromVersion) + .should.equal true + + it "should join the client to room for the doc_id", -> + @client.join + .calledWith(@doc_id) + .should.equal true + + it "should call the callback with the lines, version and ops", -> + @callback + .calledWith(null, @doc_lines, @version, @ops) + .should.equal true + + describe "with doclines that need escaping", -> + beforeEach -> + @doc_lines.push ["räksmörgås"] + @WebsocketController.joinDoc @client, @doc_id, -1, @callback + + it "should call the callback with the escaped lines", -> + escaped_lines = @callback.args[0][1] + escaped_word = escaped_lines.pop() + escaped_word.should.equal 'räksmörgÃ¥s' + # Check that unescaping works + decodeURIComponent(escape(escaped_word)).should.equal "räksmörgås" + + describe "when not authorized", -> + beforeEach -> + @AuthorizationManager.assertClientCanViewProject = sinon.stub().callsArgWith(1, @err = new Error("not authorized")) + @WebsocketController.joinDoc @client, @doc_id, -1, @callback + + it "should call the callback with an error", -> + @callback.calledWith(@err).should.equal true + + it "should not call the DocumentUpdaterManager", -> + @DocumentUpdaterManager.getDocument.called.should.equal false \ No newline at end of file From 8b923d2fda6823f2df942441da4ead59245d3f80 Mon Sep 17 00:00:00 2001 From: James Allen Date: Wed, 12 Nov 2014 16:51:48 +0000 Subject: [PATCH 006/491] Add in leaveDoc end point --- services/real-time/app/coffee/Router.coffee | 11 ++++++++++- .../app/coffee/WebsocketController.coffee | 17 ++++++++++++++--- .../unit/coffee/WebsocketControllerTests.coffee | 16 +++++++++++++++- 3 files changed, 39 insertions(+), 5 deletions(-) diff --git a/services/real-time/app/coffee/Router.coffee b/services/real-time/app/coffee/Router.coffee index 06cd7bb7a0..ac0d06f8c3 100644 --- a/services/real-time/app/coffee/Router.coffee +++ b/services/real-time/app/coffee/Router.coffee @@ -62,4 +62,13 @@ module.exports = Router = return callback {message: "Something went wrong"} else callback(null, args...) - \ No newline at end of file + + client.on "leaveDoc", (doc_id, callback) -> + WebsocketController.leaveDoc client, doc_id, (err, args...) -> + if err? + Router._getClientData client, (_, client) -> + logger.error {err, client, doc_id}, "server side error in leaveDoc" + # Don't return raw error to prevent leaking server side info + return callback {message: "Something went wrong"} + else + callback(null, args...) \ No newline at end of file diff --git a/services/real-time/app/coffee/WebsocketController.coffee b/services/real-time/app/coffee/WebsocketController.coffee index f409f39375..2f57aa4005 100644 --- a/services/real-time/app/coffee/WebsocketController.coffee +++ b/services/real-time/app/coffee/WebsocketController.coffee @@ -34,9 +34,8 @@ module.exports = WebsocketController = callback null, project, privilegeLevel, WebsocketController.PROTOCOL_VERSION joinDoc: (client, doc_id, fromVersion = -1, callback = (error, doclines, version, ops) ->) -> - client.get "user_id", (error, user_id) -> - client.get "project_id", (error, project_id) -> - logger.log {user_id, project_id, doc_id, fromVersion, client_id: client.id}, "client joining doc" + WebsocketController._getClientData client, (error, {client_id, user_id, project_id}) -> + logger.log {user_id, project_id, doc_id, fromVersion, client_id}, "client joining doc" AuthorizationManager.assertClientCanViewProject client, (error) -> return callback(error) if error? @@ -57,4 +56,16 @@ module.exports = WebsocketController = escapedLines.push line client.join(doc_id) callback null, escapedLines, version, ops + + leaveDoc: (client, doc_id, callback = (error) ->) -> + WebsocketController._getClientData client, (error, {client_id, user_id, project_id}) -> + logger.log {user_id, project_id, doc_id, client_id}, "client leaving doc" + client.leave doc_id + callback() + + # Only used in logging. + _getClientData: (client, callback = (error, data) ->) -> + client.get "user_id", (error, user_id) -> + client.get "project_id", (error, project_id) -> + callback null, {client_id: client.id, project_id, user_id} \ No newline at end of file diff --git a/services/real-time/test/unit/coffee/WebsocketControllerTests.coffee b/services/real-time/test/unit/coffee/WebsocketControllerTests.coffee index eea0e7e1f0..3be89d26e9 100644 --- a/services/real-time/test/unit/coffee/WebsocketControllerTests.coffee +++ b/services/real-time/test/unit/coffee/WebsocketControllerTests.coffee @@ -23,6 +23,7 @@ describe 'WebsocketController', -> set: sinon.stub() get: (param, cb) -> cb null, @params[param] join: sinon.stub() + leave: sinon.stub() @WebsocketController = SandboxedModule.require modulePath, requires: "./WebApiManager": @WebApiManager = {} "./AuthorizationManager": @AuthorizationManager = {} @@ -153,4 +154,17 @@ describe 'WebsocketController', -> @callback.calledWith(@err).should.equal true it "should not call the DocumentUpdaterManager", -> - @DocumentUpdaterManager.getDocument.called.should.equal false \ No newline at end of file + @DocumentUpdaterManager.getDocument.called.should.equal false + + describe "leaveDoc", -> + beforeEach -> + @doc_id = "doc-id-123" + @client.params.project_id = @project_id + @WebsocketController.leaveDoc @client, @doc_id, @callback + + it "should remove the client from the doc_id room", -> + @client.leave + .calledWith(@doc_id).should.equal true + + it "should call the callback", -> + @callback.called.should.equal true \ No newline at end of file From 0b18edeff3874f8f6972dfef53f37efd575db57e Mon Sep 17 00:00:00 2001 From: James Allen Date: Thu, 13 Nov 2014 11:48:49 +0000 Subject: [PATCH 007/491] Add in /clients and /client/:client_id status end points --- .../app/coffee/HttpController.coffee | 37 +++++++++++++++++++ services/real-time/app/coffee/Router.coffee | 7 +++- services/real-time/app/coffee/Utils.coffee | 14 +++++++ .../app/coffee/WebsocketController.coffee | 5 ++- services/real-time/package.json | 1 + .../acceptance/coffee/JoinProjectTests.coffee | 10 +++++ .../acceptance/coffee/SessionTests.coffee | 36 ++++++++++++++++-- .../coffee/helpers/RealTimeClient.coffee | 15 ++++++++ .../coffee/WebsocketControllerTests.coffee | 3 ++ 9 files changed, 122 insertions(+), 6 deletions(-) create mode 100644 services/real-time/app/coffee/HttpController.coffee create mode 100644 services/real-time/app/coffee/Utils.coffee diff --git a/services/real-time/app/coffee/HttpController.coffee b/services/real-time/app/coffee/HttpController.coffee new file mode 100644 index 0000000000..91f8df0df8 --- /dev/null +++ b/services/real-time/app/coffee/HttpController.coffee @@ -0,0 +1,37 @@ +Utils = require "./Utils" +async = require "async" + +module.exports = HttpController = + # The code in this controller is hard to unit test because of a lot of + # dependencies on internal socket.io methods. It is not critical to the running + # of ShareLaTeX, and is only used for getting stats about connected clients, + # and for checking internal state in acceptance tests. The acceptances tests + # should provide appropriate coverage. + _getConnectedClientView: (ioClient, callback = (error, client) ->) -> + client_id = ioClient.id + Utils.getClientAttributes ioClient, [ + "project_id", "user_id", "first_name", "last_name", "email", "connected_time" + ], (error, {project_id, user_id, first_name, last_name, email, connected_time}) -> + return callback(error) if error? + client = {client_id, project_id, user_id, first_name, last_name, email, connected_time} + client.rooms = [] + for name, joined of ioClient.manager.roomClients[client_id] + if joined and name != "" + client.rooms.push name.replace(/^\//, "") # Remove leading / + callback(null, client) + + getConnectedClients: (req, res, next) -> + io = req.app.get("io") + ioClients = io.sockets.clients() + async.map ioClients, HttpController._getConnectedClientView, (error, clients) -> + return next(error) if error? + res.json clients + + getConnectedClient: (req, res, next) -> + {client_id} = req.params + io = req.app.get("io") + ioClient = io.sockets.socket(client_id) + HttpController._getConnectedClientView ioClient, (error, client) -> + return next(error) if error? + res.json client + \ No newline at end of file diff --git a/services/real-time/app/coffee/Router.coffee b/services/real-time/app/coffee/Router.coffee index ac0d06f8c3..b3626f4617 100644 --- a/services/real-time/app/coffee/Router.coffee +++ b/services/real-time/app/coffee/Router.coffee @@ -1,6 +1,7 @@ Metrics = require "metrics-sharelatex" logger = require "logger-sharelatex" WebsocketController = require "./WebsocketController" +HttpController = require "./HttpController" module.exports = Router = # We don't want to send raw errors back to the client, in case they @@ -71,4 +72,8 @@ module.exports = Router = # Don't return raw error to prevent leaking server side info return callback {message: "Something went wrong"} else - callback(null, args...) \ No newline at end of file + callback(null, args...) + + app.set("io", io) + app.get "/clients", HttpController.getConnectedClients + app.get "/clients/:client_id", HttpController.getConnectedClient \ No newline at end of file diff --git a/services/real-time/app/coffee/Utils.coffee b/services/real-time/app/coffee/Utils.coffee new file mode 100644 index 0000000000..72dada30a3 --- /dev/null +++ b/services/real-time/app/coffee/Utils.coffee @@ -0,0 +1,14 @@ +async = require "async" + +module.exports = Utils = + getClientAttributes: (client, keys, callback = (error, attributes) ->) -> + attributes = {} + jobs = keys.map (key) -> + (callback) -> + client.get key, (error, value) -> + return callback(error) if error? + attributes[key] = value + callback() + async.series jobs, (error) -> + return callback(error) if error? + callback null, attributes \ No newline at end of file diff --git a/services/real-time/app/coffee/WebsocketController.coffee b/services/real-time/app/coffee/WebsocketController.coffee index 2f57aa4005..cd0e2a14dd 100644 --- a/services/real-time/app/coffee/WebsocketController.coffee +++ b/services/real-time/app/coffee/WebsocketController.coffee @@ -19,6 +19,8 @@ module.exports = WebsocketController = err = new Error("not authorized") logger.error {err, project_id, user_id, client_id: client.id}, "user is not authorized to join project" return callback(err) + + client.join project_id client.set("privilege_level", privilegeLevel) client.set("user_id", user_id) @@ -67,5 +69,4 @@ module.exports = WebsocketController = _getClientData: (client, callback = (error, data) ->) -> client.get "user_id", (error, user_id) -> client.get "project_id", (error, project_id) -> - callback null, {client_id: client.id, project_id, user_id} - \ No newline at end of file + callback null, {client_id: client.id, project_id, user_id} \ No newline at end of file diff --git a/services/real-time/package.json b/services/real-time/package.json index 5bc6c6b90e..5f9c323bb3 100644 --- a/services/real-time/package.json +++ b/services/real-time/package.json @@ -8,6 +8,7 @@ "url": "https://github.com/sharelatex/real-time-sharelatex.git" }, "dependencies": { + "async": "^0.9.0", "connect-redis": "^2.1.0", "express": "^4.10.1", "express-session": "^1.9.1", diff --git a/services/real-time/test/acceptance/coffee/JoinProjectTests.coffee b/services/real-time/test/acceptance/coffee/JoinProjectTests.coffee index 02be863b9c..36cae2b68a 100644 --- a/services/real-time/test/acceptance/coffee/JoinProjectTests.coffee +++ b/services/real-time/test/acceptance/coffee/JoinProjectTests.coffee @@ -41,6 +41,11 @@ describe "joinProject", -> it "should return the protocolVersion", -> @protocolVersion.should.equal 2 + it "should have joined the project room", (done) -> + RealTimeClient.getConnectedClient @client.socket.sessionid, (error, client) => + expect(@project_id in client.rooms).to.equal true + done() + describe "when not authorized", -> before (done) -> FixturesManager.setUpProject { @@ -60,3 +65,8 @@ describe "joinProject", -> it "should return an error", -> # We don't return specific errors @error.message.should.equal "Something went wrong" + + it "should not have joined the project room", (done) -> + RealTimeClient.getConnectedClient @client.socket.sessionid, (error, client) => + expect(@project_id in client.rooms).to.equal false + done() diff --git a/services/real-time/test/acceptance/coffee/SessionTests.coffee b/services/real-time/test/acceptance/coffee/SessionTests.coffee index 2eb0d8206a..618635db58 100644 --- a/services/real-time/test/acceptance/coffee/SessionTests.coffee +++ b/services/real-time/test/acceptance/coffee/SessionTests.coffee @@ -5,7 +5,7 @@ RealTimeClient = require "./helpers/RealTimeClient" describe "Session", -> describe "with an established session", -> - beforeEach (done) -> + before (done) -> @user_id = "mock-user-id" RealTimeClient.setSession { user: { _id: @user_id } @@ -22,9 +22,19 @@ describe "Session", -> expect(disconnected).to.equal false done() , 500 + + it "should appear in the list of connected clients", (done) -> + RealTimeClient.getConnectedClients (error, clients) => + included = false + for client in clients + if client.client_id == @client.socket.sessionid + included = true + break + expect(included).to.equal true + done() describe "without an established session", -> - beforeEach (done) -> + before (done) -> RealTimeClient.unsetSession (error) => throw error if error? @client = RealTimeClient.connect() @@ -34,8 +44,18 @@ describe "Session", -> @client.on "disconnect", () -> done() + it "not should appear in the list of connected clients", (done) -> + RealTimeClient.getConnectedClients (error, clients) => + included = false + for client in clients + if client.client_id == @client.socket.sessionid + included = true + break + expect(included).to.equal false + done() + describe "without a valid user set on the session", -> - beforeEach (done) -> + before (done) -> RealTimeClient.setSession { foo: "bar" }, (error) => @@ -45,4 +65,14 @@ describe "Session", -> it "should get disconnected", (done) -> @client.on "disconnect", () -> + done() + + it "not should appear in the list of connected clients", (done) -> + RealTimeClient.getConnectedClients (error, clients) => + included = false + for client in clients + if client.client_id == @client.socket.sessionid + included = true + break + expect(included).to.equal false done() \ No newline at end of file diff --git a/services/real-time/test/acceptance/coffee/helpers/RealTimeClient.coffee b/services/real-time/test/acceptance/coffee/helpers/RealTimeClient.coffee index 415f14fe5e..52b869f862 100644 --- a/services/real-time/test/acceptance/coffee/helpers/RealTimeClient.coffee +++ b/services/real-time/test/acceptance/coffee/helpers/RealTimeClient.coffee @@ -1,6 +1,7 @@ XMLHttpRequest = require("../../libs/XMLHttpRequest").XMLHttpRequest io = require("socket.io-client") +request = require "request" Settings = require "settings-sharelatex" redis = require "redis-sharelatex" rclient = redis.createClient(Settings.redis.web) @@ -37,4 +38,18 @@ module.exports = Client = connect: (cookie) -> client = io.connect("http://localhost:3026", 'force new connection': true) return client + + getConnectedClients: (callback = (error, clients) ->) -> + request.get { + url: "http://localhost:3026/clients" + json: true + }, (error, response, data) -> + callback error, data + + getConnectedClient: (client_id, callback = (error, clients) ->) -> + request.get { + url: "http://localhost:3026/clients/#{client_id}" + json: true + }, (error, response, data) -> + callback error, data diff --git a/services/real-time/test/unit/coffee/WebsocketControllerTests.coffee b/services/real-time/test/unit/coffee/WebsocketControllerTests.coffee index 3be89d26e9..0a0fc36dcb 100644 --- a/services/real-time/test/unit/coffee/WebsocketControllerTests.coffee +++ b/services/real-time/test/unit/coffee/WebsocketControllerTests.coffee @@ -51,6 +51,9 @@ describe 'WebsocketController', -> .calledWith(@project_id, @user._id) .should.equal true + it "should join the project room", -> + @client.join.calledWith(@project_id).should.equal true + it "should set the privilege level on the client", -> @client.set.calledWith("privilege_level", @privilegeLevel).should.equal true From 431abdc6eba106b6d6ba20d108de8487a3c9eb17 Mon Sep 17 00:00:00 2001 From: James Allen Date: Thu, 13 Nov 2014 11:54:10 +0000 Subject: [PATCH 008/491] Add leaveDoc acceptance tests --- .../acceptance/coffee/JoinDocTests.coffee | 22 +++++++++- .../acceptance/coffee/LeaveDocTests.coffee | 41 +++++++++++++++++++ 2 files changed, 62 insertions(+), 1 deletion(-) create mode 100644 services/real-time/test/acceptance/coffee/LeaveDocTests.coffee diff --git a/services/real-time/test/acceptance/coffee/JoinDocTests.coffee b/services/real-time/test/acceptance/coffee/JoinDocTests.coffee index a339cf7ef8..707108ebf6 100644 --- a/services/real-time/test/acceptance/coffee/JoinDocTests.coffee +++ b/services/real-time/test/acceptance/coffee/JoinDocTests.coffee @@ -36,6 +36,11 @@ describe "joinDoc", -> it "should return the doc lines, version and ops", -> @returnedArgs.should.deep.equal [@lines, @version, @ops] + + it "should have joined the doc room", (done) -> + RealTimeClient.getConnectedClient @client.socket.sessionid, (error, client) => + expect(@doc_id in client.rooms).to.equal true + done() describe "when authorised readOnly", -> before (done) -> @@ -61,6 +66,11 @@ describe "joinDoc", -> it "should return the doc lines, version and ops", -> @returnedArgs.should.deep.equal [@lines, @version, @ops] + + it "should have joined the doc room", (done) -> + RealTimeClient.getConnectedClient @client.socket.sessionid, (error, client) => + expect(@doc_id in client.rooms).to.equal true + done() describe "when authorised as owner", -> before (done) -> @@ -86,6 +96,11 @@ describe "joinDoc", -> it "should return the doc lines, version and ops", -> @returnedArgs.should.deep.equal [@lines, @version, @ops] + + it "should have joined the doc room", (done) -> + RealTimeClient.getConnectedClient @client.socket.sessionid, (error, client) => + expect(@doc_id in client.rooms).to.equal true + done() # It is impossible to write an acceptance test to test joining an unauthorized # project, since joinProject already catches that. If you can join a project, @@ -115,4 +130,9 @@ describe "joinDoc", -> .should.equal true it "should return the doc lines, version and ops", -> - @returnedArgs.should.deep.equal [@lines, @version, @ops] \ No newline at end of file + @returnedArgs.should.deep.equal [@lines, @version, @ops] + + it "should have joined the doc room", (done) -> + RealTimeClient.getConnectedClient @client.socket.sessionid, (error, client) => + expect(@doc_id in client.rooms).to.equal true + done() \ No newline at end of file diff --git a/services/real-time/test/acceptance/coffee/LeaveDocTests.coffee b/services/real-time/test/acceptance/coffee/LeaveDocTests.coffee new file mode 100644 index 0000000000..fe1ef94f4d --- /dev/null +++ b/services/real-time/test/acceptance/coffee/LeaveDocTests.coffee @@ -0,0 +1,41 @@ +chai = require("chai") +expect = chai.expect +chai.should() + +RealTimeClient = require "./helpers/RealTimeClient" +MockDocUpdaterServer = require "./helpers/MockDocUpdaterServer" +FixturesManager = require "./helpers/FixturesManager" + +describe "leaveDoc", -> + before -> + @lines = ["test", "doc", "lines"] + @version = 42 + @ops = ["mock", "doc", "ops"] + + describe "when joined to a doc", -> + before (done) -> + FixturesManager.setUpProject { + privilegeLevel: "readAndWrite" + }, (error, data) => + throw error if error? + {@project_id, @user_id} = data + FixturesManager.setUpDoc @project_id, {@lines, @version, @ops}, (error, data) => + throw error if error? + {@doc_id} = data + @client = RealTimeClient.connect() + @client.emit "joinProject", project_id: @project_id, (error) => + throw error if error? + @client.emit "joinDoc", @doc_id, (error, @returnedArgs...) => + throw error if error? + done() + + describe "then leaving the doc", -> + before (done) -> + @client.emit "leaveDoc", @doc_id, (error) -> + throw error if error? + done() + + it "should have left the doc room", (done) -> + RealTimeClient.getConnectedClient @client.socket.sessionid, (error, client) => + expect(@doc_id in client.rooms).to.equal false + done() From 6ed2a0d04d56197f97fd875fae6d8709f9eb2c16 Mon Sep 17 00:00:00 2001 From: James Allen Date: Thu, 13 Nov 2014 12:03:43 +0000 Subject: [PATCH 009/491] Refactor client attribute fetching and logging --- services/real-time/app/coffee/Router.coffee | 38 ++++++------------- .../app/coffee/WebsocketController.coffee | 15 +++----- 2 files changed, 16 insertions(+), 37 deletions(-) diff --git a/services/real-time/app/coffee/Router.coffee b/services/real-time/app/coffee/Router.coffee index b3626f4617..58baa28af9 100644 --- a/services/real-time/app/coffee/Router.coffee +++ b/services/real-time/app/coffee/Router.coffee @@ -2,26 +2,14 @@ Metrics = require "metrics-sharelatex" logger = require "logger-sharelatex" WebsocketController = require "./WebsocketController" HttpController = require "./HttpController" +Utils = require "./Utils" module.exports = Router = - # We don't want to send raw errors back to the client, in case they - # contain sensitive data. Instead we log them out, and send a generic - # JSON object which can be serialized over socket.io - _createCallbackWithErrorFilter: (client, method, callback) -> - return (err, args...) -> - if err? - - err = {message: "Something went wrong"} - callback err, args... - - # Used in error reporting - _getClientData: (client, callback = (error, data) ->) -> - client.get "user_id", (error, user_id) -> - client.get "project_id", (error, project_id) -> - client.get "doc_id", (error, doc_id) -> - callback null, { id: client.id, user_id, project_id, doc_id } - configure: (app, io, session) -> + app.set("io", io) + app.get "/clients", HttpController.getConnectedClients + app.get "/clients/:client_id", HttpController.getConnectedClient + session.on 'connection', (error, client, session) -> if error? logger.err err: error, "error when client connected" @@ -41,8 +29,7 @@ module.exports = Router = client.on "joinProject", (data = {}, callback) -> WebsocketController.joinProject client, user, data.project_id, (err, args...) -> if err? - Router._getClientData client, (_, client) -> - logger.error {err, client, project_id: data.project_id}, "server side error in joinProject" + logger.error {err, user_id: user?.id, project_id: data.project_id}, "server side error in joinProject" # Don't return raw error to prevent leaking server side info return callback {message: "Something went wrong"} else @@ -57,8 +44,8 @@ module.exports = Router = WebsocketController.joinDoc client, doc_id, fromVersion, (err, args...) -> if err? - Router._getClientData client, (_, client) -> - logger.error {err, client, doc_id, fromVersion}, "server side error in joinDoc" + Utils.getClientAttributes client, ["project_id", "user_id"], (_, {project_id, user_id}) -> + logger.error {err, client_id: client.id, user_id, project_id, doc_id, fromVersion}, "server side error in joinDoc" # Don't return raw error to prevent leaking server side info return callback {message: "Something went wrong"} else @@ -67,13 +54,10 @@ module.exports = Router = client.on "leaveDoc", (doc_id, callback) -> WebsocketController.leaveDoc client, doc_id, (err, args...) -> if err? - Router._getClientData client, (_, client) -> - logger.error {err, client, doc_id}, "server side error in leaveDoc" + Utils.getClientAttributes client, ["project_id", "user_id"], (_, {project_id, user_id}) -> + logger.error {err, client_id: client.id, user_id, project_id, doc_id}, "server side error in leaveDoc" # Don't return raw error to prevent leaking server side info return callback {message: "Something went wrong"} else callback(null, args...) - - app.set("io", io) - app.get "/clients", HttpController.getConnectedClients - app.get "/clients/:client_id", HttpController.getConnectedClient \ No newline at end of file + \ No newline at end of file diff --git a/services/real-time/app/coffee/WebsocketController.coffee b/services/real-time/app/coffee/WebsocketController.coffee index cd0e2a14dd..30ea76bebf 100644 --- a/services/real-time/app/coffee/WebsocketController.coffee +++ b/services/real-time/app/coffee/WebsocketController.coffee @@ -2,6 +2,7 @@ logger = require "logger-sharelatex" WebApiManager = require "./WebApiManager" AuthorizationManager = require "./AuthorizationManager" DocumentUpdaterManager = require "./DocumentUpdaterManager" +Utils = require "./Utils" module.exports = WebsocketController = # If the protocol version changes when the client reconnects, @@ -36,8 +37,8 @@ module.exports = WebsocketController = callback null, project, privilegeLevel, WebsocketController.PROTOCOL_VERSION joinDoc: (client, doc_id, fromVersion = -1, callback = (error, doclines, version, ops) ->) -> - WebsocketController._getClientData client, (error, {client_id, user_id, project_id}) -> - logger.log {user_id, project_id, doc_id, fromVersion, client_id}, "client joining doc" + Utils.getClientAttributes client, ["project_id", "user_id"], (error, {project_id, user_id}) -> + logger.log {user_id, project_id, doc_id, fromVersion, client_id: client.id}, "client joining doc" AuthorizationManager.assertClientCanViewProject client, (error) -> return callback(error) if error? @@ -60,13 +61,7 @@ module.exports = WebsocketController = callback null, escapedLines, version, ops leaveDoc: (client, doc_id, callback = (error) ->) -> - WebsocketController._getClientData client, (error, {client_id, user_id, project_id}) -> - logger.log {user_id, project_id, doc_id, client_id}, "client leaving doc" + Utils.getClientAttributes client, ["project_id", "user_id"], (error, {project_id, user_id}) -> + logger.log {user_id, project_id, doc_id, client_id: client.id}, "client leaving doc" client.leave doc_id callback() - - # Only used in logging. - _getClientData: (client, callback = (error, data) ->) -> - client.get "user_id", (error, user_id) -> - client.get "project_id", (error, project_id) -> - callback null, {client_id: client.id, project_id, user_id} \ No newline at end of file From f7482014ce358328b4d9d805bba0b4903944d8dc Mon Sep 17 00:00:00 2001 From: James Allen Date: Thu, 13 Nov 2014 12:27:46 +0000 Subject: [PATCH 010/491] Import ConnectedUsersManager from web --- .../app/coffee/ConnectedUsersManager.coffee | 78 +++++++++ .../coffee/ConnectedUsersManagerTests.coffee | 154 ++++++++++++++++++ 2 files changed, 232 insertions(+) create mode 100644 services/real-time/app/coffee/ConnectedUsersManager.coffee create mode 100644 services/real-time/test/unit/coffee/ConnectedUsersManagerTests.coffee diff --git a/services/real-time/app/coffee/ConnectedUsersManager.coffee b/services/real-time/app/coffee/ConnectedUsersManager.coffee new file mode 100644 index 0000000000..716a8a2bdf --- /dev/null +++ b/services/real-time/app/coffee/ConnectedUsersManager.coffee @@ -0,0 +1,78 @@ +async = require("async") +Settings = require('settings-sharelatex') +logger = require("logger-sharelatex") +redis = require("redis-sharelatex") +rclient = redis.createClient(Settings.redis.web) + + +ONE_HOUR_IN_S = 60 * 60 +ONE_DAY_IN_S = ONE_HOUR_IN_S * 24 +FOUR_DAYS_IN_S = ONE_DAY_IN_S * 4 + +USER_TIMEOUT_IN_S = ONE_HOUR_IN_S / 4 + +buildProjectSetKey = (project_id)-> return "clients_in_project:#{project_id}" +buildUserKey = (project_id, client_id)-> return "connected_user:#{project_id}:#{client_id}" + + +module.exports = + + # Use the same method for when a user connects, and when a user sends a cursor + # update. This way we don't care if the connected_user key has expired when + # we receive a cursor update. + updateUserPosition: (project_id, client_id, user, cursorData, callback = (err)->)-> + logger.log project_id:project_id, client_id:client_id, "marking user as connected" + + multi = rclient.multi() + + multi.sadd buildProjectSetKey(project_id), client_id + multi.expire buildProjectSetKey(project_id), FOUR_DAYS_IN_S + + multi.hset buildUserKey(project_id, client_id), "last_updated_at", Date.now() + multi.hset buildUserKey(project_id, client_id), "user_id", user._id + multi.hset buildUserKey(project_id, client_id), "first_name", user.first_name + multi.hset buildUserKey(project_id, client_id), "last_name", user.last_name + multi.hset buildUserKey(project_id, client_id), "email", user.email + + if cursorData? + multi.hset buildUserKey(project_id, client_id), "cursorData", JSON.stringify(cursorData) + multi.expire buildUserKey(project_id, client_id), USER_TIMEOUT_IN_S + + multi.exec (err)-> + if err? + logger.err err:err, project_id:project_id, client_id:client_id, "problem marking user as connected" + callback(err) + + markUserAsDisconnected: (project_id, client_id, callback)-> + logger.log project_id:project_id, client_id:client_id, "marking user as disconnected" + multi = rclient.multi() + multi.srem buildProjectSetKey(project_id), client_id + multi.expire buildProjectSetKey(project_id), FOUR_DAYS_IN_S + multi.del buildUserKey(project_id, client_id) + multi.exec callback + + + _getConnectedUser: (project_id, client_id, callback)-> + rclient.hgetall buildUserKey(project_id, client_id), (err, result)-> + if !result? + result = + connected : false + client_id:client_id + else + result.connected = true + result.client_id = client_id + if result.cursorData? + result.cursorData = JSON.parse(result.cursorData) + callback err, result + + getConnectedUsers: (project_id, callback)-> + self = @ + rclient.smembers buildProjectSetKey(project_id), (err, results)-> + jobs = results.map (client_id)-> + (cb)-> + self._getConnectedUser(project_id, client_id, cb) + async.series jobs, (err, users)-> + users = users.filter (user) -> + user.connected + callback err, users + diff --git a/services/real-time/test/unit/coffee/ConnectedUsersManagerTests.coffee b/services/real-time/test/unit/coffee/ConnectedUsersManagerTests.coffee new file mode 100644 index 0000000000..982f4a170f --- /dev/null +++ b/services/real-time/test/unit/coffee/ConnectedUsersManagerTests.coffee @@ -0,0 +1,154 @@ + +should = require('chai').should() +SandboxedModule = require('sandboxed-module') +assert = require('assert') +path = require('path') +sinon = require('sinon') +modulePath = path.join __dirname, "../../../app/js/ConnectedUsersManager" +expect = require("chai").expect +tk = require("timekeeper") + + +describe "ConnectedUsersManager", -> + + beforeEach -> + + @settings = + redis: + web:{} + @rClient = + auth:-> + setex:sinon.stub() + sadd:sinon.stub() + get: sinon.stub() + srem:sinon.stub() + del:sinon.stub() + smembers:sinon.stub() + expire:sinon.stub() + hset:sinon.stub() + hgetall:sinon.stub() + exec:sinon.stub() + multi: => return @rClient + tk.freeze(new Date()) + + @ConnectedUsersManager = SandboxedModule.require modulePath, requires: + "settings-sharelatex":@settings + "logger-sharelatex": log:-> + "redis-sharelatex": createClient:=> + return @rClient + @client_id = "32132132" + @project_id = "dskjh2u21321" + @user = { + _id: "user-id-123" + first_name: "Joe" + last_name: "Bloggs" + email: "joe@example.com" + } + @cursorData = { row: 12, column: 9, doc_id: '53c3b8c85fee64000023dc6e' } + + afterEach -> + tk.reset() + + describe "updateUserPosition", -> + beforeEach -> + @rClient.exec.callsArgWith(0) + + it "should set a key with the date and give it a ttl", (done)-> + @ConnectedUsersManager.updateUserPosition @project_id, @client_id, @user, null, (err)=> + @rClient.hset.calledWith("connected_user:#{@project_id}:#{@client_id}", "last_updated_at", Date.now()).should.equal true + done() + + it "should set a key with the user_id", (done)-> + @ConnectedUsersManager.updateUserPosition @project_id, @client_id, @user, null, (err)=> + @rClient.hset.calledWith("connected_user:#{@project_id}:#{@client_id}", "user_id", @user._id).should.equal true + done() + + it "should set a key with the first_name", (done)-> + @ConnectedUsersManager.updateUserPosition @project_id, @client_id, @user, null, (err)=> + @rClient.hset.calledWith("connected_user:#{@project_id}:#{@client_id}", "first_name", @user.first_name).should.equal true + done() + + it "should set a key with the last_name", (done)-> + @ConnectedUsersManager.updateUserPosition @project_id, @client_id, @user, null, (err)=> + @rClient.hset.calledWith("connected_user:#{@project_id}:#{@client_id}", "last_name", @user.last_name).should.equal true + done() + + it "should set a key with the email", (done)-> + @ConnectedUsersManager.updateUserPosition @project_id, @client_id, @user, null, (err)=> + @rClient.hset.calledWith("connected_user:#{@project_id}:#{@client_id}", "email", @user.email).should.equal true + done() + + it "should push the client_id on to the project list", (done)-> + @ConnectedUsersManager.updateUserPosition @project_id, @client_id, @user, null, (err)=> + @rClient.sadd.calledWith("clients_in_project:#{@project_id}", @client_id).should.equal true + done() + + it "should add a ttl to the project set so it stays clean", (done)-> + @ConnectedUsersManager.updateUserPosition @project_id, @client_id, @user, null, (err)=> + @rClient.expire.calledWith("clients_in_project:#{@project_id}", 24 * 4 * 60 * 60).should.equal true + done() + + it "should add a ttl to the connected user so it stays clean", (done) -> + @ConnectedUsersManager.updateUserPosition @project_id, @client_id, @user, null, (err)=> + @rClient.expire.calledWith("connected_user:#{@project_id}:#{@client_id}", 60 * 15).should.equal true + done() + + it "should set the cursor position when provided", (done)-> + @ConnectedUsersManager.updateUserPosition @project_id, @client_id, @user, @cursorData, (err)=> + @rClient.hset.calledWith("connected_user:#{@project_id}:#{@client_id}", "cursorData", JSON.stringify(@cursorData)).should.equal true + done() + + describe "markUserAsDisconnected", -> + beforeEach -> + @rClient.exec.callsArgWith(0) + + it "should remove the user from the set", (done)-> + @ConnectedUsersManager.markUserAsDisconnected @project_id, @client_id, (err)=> + @rClient.srem.calledWith("clients_in_project:#{@project_id}", @client_id).should.equal true + done() + + it "should delete the connected_user string", (done)-> + @ConnectedUsersManager.markUserAsDisconnected @project_id, @client_id, (err)=> + @rClient.del.calledWith("connected_user:#{@project_id}:#{@client_id}").should.equal true + done() + + it "should add a ttl to the connected user set so it stays clean", (done)-> + @ConnectedUsersManager.markUserAsDisconnected @project_id, @client_id, (err)=> + @rClient.expire.calledWith("clients_in_project:#{@project_id}", 24 * 4 * 60 * 60).should.equal true + done() + + describe "_getConnectedUser", -> + + it "should get the user returning connected if there is a value", (done)-> + cursorData = JSON.stringify(cursorData:{row:1}) + @rClient.hgetall.callsArgWith(1, null, {connected_at:new Date(), cursorData}) + @ConnectedUsersManager._getConnectedUser @project_id, @client_id, (err, result)=> + result.connected.should.equal true + result.client_id.should.equal @client_id + done() + + it "should get the user returning connected if there is a value", (done)-> + @rClient.hgetall.callsArgWith(1) + @ConnectedUsersManager._getConnectedUser @project_id, @client_id, (err, result)=> + result.connected.should.equal false + result.client_id.should.equal @client_id + done() + + describe "getConnectedUsers", -> + + beforeEach -> + @users = ["1234", "5678", "9123"] + @rClient.smembers.callsArgWith(1, null, @users) + @ConnectedUsersManager._getConnectedUser = sinon.stub() + @ConnectedUsersManager._getConnectedUser.withArgs(@project_id, @users[0]).callsArgWith(2, null, {connected:true, client_id:@users[0]}) + @ConnectedUsersManager._getConnectedUser.withArgs(@project_id, @users[1]).callsArgWith(2, null, {connected:false, client_id:@users[1]}) + @ConnectedUsersManager._getConnectedUser.withArgs(@project_id, @users[2]).callsArgWith(2, null, {connected:true, client_id:@users[2]}) + + + it "should only return the users in the list which are still in redis", (done)-> + @ConnectedUsersManager.getConnectedUsers @project_id, (err, users)=> + users.length.should.equal 2 + users[0].should.deep.equal {client_id:@users[0], connected:true} + users[1].should.deep.equal {client_id:@users[2], connected:true} + done() + From 84778b59618748aaa160729670d90bc53105c489 Mon Sep 17 00:00:00 2001 From: James Allen Date: Thu, 13 Nov 2014 13:05:49 +0000 Subject: [PATCH 011/491] Mark user as connected for cursor updates when joining project --- .../app/coffee/ConnectedUsersManager.coffee | 23 ++++++--- services/real-time/app/coffee/Router.coffee | 10 ++++ .../app/coffee/WebsocketController.coffee | 14 ++++++ .../acceptance/coffee/JoinProjectTests.coffee | 10 ++++ .../coffee/WebsocketControllerTests.coffee | 49 ++++++++++++++++++- 5 files changed, 97 insertions(+), 9 deletions(-) diff --git a/services/real-time/app/coffee/ConnectedUsersManager.coffee b/services/real-time/app/coffee/ConnectedUsersManager.coffee index 716a8a2bdf..5a77660f55 100644 --- a/services/real-time/app/coffee/ConnectedUsersManager.coffee +++ b/services/real-time/app/coffee/ConnectedUsersManager.coffee @@ -21,7 +21,7 @@ module.exports = # update. This way we don't care if the connected_user key has expired when # we receive a cursor update. updateUserPosition: (project_id, client_id, user, cursorData, callback = (err)->)-> - logger.log project_id:project_id, client_id:client_id, "marking user as connected" + logger.log project_id:project_id, client_id:client_id, "marking user as joined or connected" multi = rclient.multi() @@ -30,9 +30,9 @@ module.exports = multi.hset buildUserKey(project_id, client_id), "last_updated_at", Date.now() multi.hset buildUserKey(project_id, client_id), "user_id", user._id - multi.hset buildUserKey(project_id, client_id), "first_name", user.first_name - multi.hset buildUserKey(project_id, client_id), "last_name", user.last_name - multi.hset buildUserKey(project_id, client_id), "email", user.email + multi.hset buildUserKey(project_id, client_id), "first_name", user.first_name or "" + multi.hset buildUserKey(project_id, client_id), "last_name", user.last_name or "" + multi.hset buildUserKey(project_id, client_id), "email", user.email or "" if cursorData? multi.hset buildUserKey(project_id, client_id), "cursorData", JSON.stringify(cursorData) @@ -61,18 +61,25 @@ module.exports = else result.connected = true result.client_id = client_id + console.log "RESULT", result if result.cursorData? - result.cursorData = JSON.parse(result.cursorData) + try + result.cursorData = JSON.parse(result.cursorData) + catch e + logger.error {err: e, project_id, client_id, cursorData: result.cursorData}, "error parsing cursorData JSON" + return callback e callback err, result getConnectedUsers: (project_id, callback)-> self = @ rclient.smembers buildProjectSetKey(project_id), (err, results)-> + return callback(err) if err? jobs = results.map (client_id)-> (cb)-> self._getConnectedUser(project_id, client_id, cb) - async.series jobs, (err, users)-> + async.series jobs, (err, users = [])-> + return callback(err) if err? users = users.filter (user) -> - user.connected - callback err, users + user?.connected + callback null, users diff --git a/services/real-time/app/coffee/Router.coffee b/services/real-time/app/coffee/Router.coffee index 58baa28af9..f8aedbf46b 100644 --- a/services/real-time/app/coffee/Router.coffee +++ b/services/real-time/app/coffee/Router.coffee @@ -60,4 +60,14 @@ module.exports = Router = return callback {message: "Something went wrong"} else callback(null, args...) + + client.on "getConnectedUsers", (callback = (error, users) ->) -> + WebsocketController.getConnectedUsers client, (err, users) -> + if err? + Utils.getClientAttributes client, ["project_id", "user_id", "doc_id"], (_, {project_id, user_id, doc_id}) -> + logger.error {err, client_id: client.id, user_id, project_id, doc_id}, "server side error in getConnectedUsers" + # Don't return raw error to prevent leaking server side info + return callback {message: "Something went wrong"} + else + callback(null, users) \ No newline at end of file diff --git a/services/real-time/app/coffee/WebsocketController.coffee b/services/real-time/app/coffee/WebsocketController.coffee index 30ea76bebf..7fa4ca6f01 100644 --- a/services/real-time/app/coffee/WebsocketController.coffee +++ b/services/real-time/app/coffee/WebsocketController.coffee @@ -2,6 +2,7 @@ logger = require "logger-sharelatex" WebApiManager = require "./WebApiManager" AuthorizationManager = require "./AuthorizationManager" DocumentUpdaterManager = require "./DocumentUpdaterManager" +ConnectedUsersManager = require "./ConnectedUsersManager" Utils = require "./Utils" module.exports = WebsocketController = @@ -36,6 +37,9 @@ module.exports = WebsocketController = callback null, project, privilegeLevel, WebsocketController.PROTOCOL_VERSION + # No need to block for setting the user as connected in the cursor tracking + ConnectedUsersManager.updateUserPosition project_id, client.id, user, null, () -> + joinDoc: (client, doc_id, fromVersion = -1, callback = (error, doclines, version, ops) ->) -> Utils.getClientAttributes client, ["project_id", "user_id"], (error, {project_id, user_id}) -> logger.log {user_id, project_id, doc_id, fromVersion, client_id: client.id}, "client joining doc" @@ -65,3 +69,13 @@ module.exports = WebsocketController = logger.log {user_id, project_id, doc_id, client_id: client.id}, "client leaving doc" client.leave doc_id callback() + + getConnectedUsers: (client, callback = (error, users) ->) -> + AuthorizationManager.assertClientCanViewProject client, (error) -> + return callback(error) if error? + client.get "project_id", (error, project_id) -> + return callback(error) if error? + return callback(new Error("no project_id found on client")) if !project_id? + ConnectedUsersManager.getConnectedUsers project_id, (error, users) -> + return callback(error) if error? + callback null, users diff --git a/services/real-time/test/acceptance/coffee/JoinProjectTests.coffee b/services/real-time/test/acceptance/coffee/JoinProjectTests.coffee index 36cae2b68a..48b8732080 100644 --- a/services/real-time/test/acceptance/coffee/JoinProjectTests.coffee +++ b/services/real-time/test/acceptance/coffee/JoinProjectTests.coffee @@ -45,6 +45,16 @@ describe "joinProject", -> RealTimeClient.getConnectedClient @client.socket.sessionid, (error, client) => expect(@project_id in client.rooms).to.equal true done() + + it "should have marked the user as connected", (done) -> + @client.emit "getConnectedUsers", (error, users) => + connected = false + for user in users + if user.client_id == @client.socket.sessionid and user.user_id == @user_id + connected = true + break + expect(connected).to.equal true + done() describe "when not authorized", -> before (done) -> diff --git a/services/real-time/test/unit/coffee/WebsocketControllerTests.coffee b/services/real-time/test/unit/coffee/WebsocketControllerTests.coffee index 0a0fc36dcb..976e113291 100644 --- a/services/real-time/test/unit/coffee/WebsocketControllerTests.coffee +++ b/services/real-time/test/unit/coffee/WebsocketControllerTests.coffee @@ -28,6 +28,7 @@ describe 'WebsocketController', -> "./WebApiManager": @WebApiManager = {} "./AuthorizationManager": @AuthorizationManager = {} "./DocumentUpdaterManager": @DocumentUpdaterManager = {} + "./ConnectedUsersManager": @ConnectedUsersManager = {} "logger-sharelatex": @logger = { log: sinon.stub(), error: sinon.stub() } afterEach -> @@ -36,6 +37,7 @@ describe 'WebsocketController', -> describe "joinProject", -> describe "when authorised", -> beforeEach -> + @client.id = "mock-client-id" @project = { name: "Test Project" owner: { @@ -43,6 +45,7 @@ describe 'WebsocketController', -> } } @privilegeLevel = "owner" + @ConnectedUsersManager.updateUserPosition = sinon.stub().callsArg(4) @WebApiManager.joinProject = sinon.stub().callsArgWith(2, null, @project, @privilegeLevel) @WebsocketController.joinProject @client, @user, @project_id, @callback @@ -88,6 +91,11 @@ describe 'WebsocketController', -> @callback .calledWith(null, @project, @privilegeLevel, @WebsocketController.PROTOCOL_VERSION) .should.equal true + + it "should mark the user as connected in ConnectedUsersManager", -> + @ConnectedUsersManager.updateUserPosition + .calledWith(@project_id, @client.id, @user, null) + .should.equal true describe "when not authorized", -> beforeEach -> @@ -170,4 +178,43 @@ describe 'WebsocketController', -> .calledWith(@doc_id).should.equal true it "should call the callback", -> - @callback.called.should.equal true \ No newline at end of file + @callback.called.should.equal true + + describe "getConnectedUsers", -> + beforeEach -> + @client.params.project_id = @project_id + @users = ["mock", "users"] + @ConnectedUsersManager.getConnectedUsers = sinon.stub().callsArgWith(1, null, @users) + + describe "when authorized", -> + beforeEach -> + @AuthorizationManager.assertClientCanViewProject = sinon.stub().callsArgWith(1, null) + @WebsocketController.getConnectedUsers @client, @callback + + it "should check that the client is authorized to view the project", -> + @AuthorizationManager.assertClientCanViewProject + .calledWith(@client) + .should.equal true + + it "should get the connected users for the project", -> + @ConnectedUsersManager.getConnectedUsers + .calledWith(@project_id) + .should.equal true + + it "should return the users", -> + @callback.calledWith(null, @users).should.equal true + + describe "when not authorized", -> + beforeEach -> + @AuthorizationManager.assertClientCanViewProject = sinon.stub().callsArgWith(1, @err = new Error("not authorized")) + @WebsocketController.getConnectedUsers @client, @callback + + it "should not get the connected users for the project", -> + @ConnectedUsersManager.getConnectedUsers + .called + .should.equal false + + it "should return an error", -> + @callback.calledWith(@err).should.equal true + + \ No newline at end of file From e76981952182dc5e81bc158ade1e05d1b147e9b8 Mon Sep 17 00:00:00 2001 From: James Allen Date: Thu, 13 Nov 2014 15:27:18 +0000 Subject: [PATCH 012/491] Add in clientTracking.updatePosition end point --- services/real-time/app/coffee/Router.coffee | 39 ++++++----- .../app/coffee/WebsocketController.coffee | 30 ++++++++ .../coffee/ClientTrackingTests.coffee | 61 ++++++++++++++++ .../acceptance/coffee/JoinProjectTests.coffee | 2 +- .../coffee/helpers/FixturesManager.coffee | 6 +- .../coffee/WebsocketControllerTests.coffee | 70 ++++++++++++++++++- 6 files changed, 188 insertions(+), 20 deletions(-) create mode 100644 services/real-time/test/acceptance/coffee/ClientTrackingTests.coffee diff --git a/services/real-time/app/coffee/Router.coffee b/services/real-time/app/coffee/Router.coffee index f8aedbf46b..43ec7976f0 100644 --- a/services/real-time/app/coffee/Router.coffee +++ b/services/real-time/app/coffee/Router.coffee @@ -5,6 +5,16 @@ HttpController = require "./HttpController" Utils = require "./Utils" module.exports = Router = + _handleError: (callback, error, client, method, extraAttrs = {}) -> + Utils.getClientAttributes client, ["project_id", "doc_id", "user_id"], (_, attrs) -> + for key, value of extraAttrs + attrs[key] = value + attrs.client_id = client.id + attrs.err = error + logger.error attrs, "server side error in #{method}" + # Don't return raw error to prevent leaking server side info + return callback {message: "Something went wrong"} + configure: (app, io, session) -> app.set("io", io) app.get "/clients", HttpController.getConnectedClients @@ -29,9 +39,7 @@ module.exports = Router = client.on "joinProject", (data = {}, callback) -> WebsocketController.joinProject client, user, data.project_id, (err, args...) -> if err? - logger.error {err, user_id: user?.id, project_id: data.project_id}, "server side error in joinProject" - # Don't return raw error to prevent leaking server side info - return callback {message: "Something went wrong"} + Router._handleError callback, err, client, "joinProject", {project_id: data.project_id, user_id: user?.id} else callback(null, args...) @@ -44,30 +52,27 @@ module.exports = Router = WebsocketController.joinDoc client, doc_id, fromVersion, (err, args...) -> if err? - Utils.getClientAttributes client, ["project_id", "user_id"], (_, {project_id, user_id}) -> - logger.error {err, client_id: client.id, user_id, project_id, doc_id, fromVersion}, "server side error in joinDoc" - # Don't return raw error to prevent leaking server side info - return callback {message: "Something went wrong"} + Router._handleError callback, err, client, "joinDoc", {doc_id, fromVersion} else callback(null, args...) client.on "leaveDoc", (doc_id, callback) -> WebsocketController.leaveDoc client, doc_id, (err, args...) -> if err? - Utils.getClientAttributes client, ["project_id", "user_id"], (_, {project_id, user_id}) -> - logger.error {err, client_id: client.id, user_id, project_id, doc_id}, "server side error in leaveDoc" - # Don't return raw error to prevent leaking server side info - return callback {message: "Something went wrong"} + Router._handleError callback, err, client, "leaveDoc" else callback(null, args...) - client.on "getConnectedUsers", (callback = (error, users) ->) -> + client.on "clientTracking.getConnectedUsers", (callback = (error, users) ->) -> WebsocketController.getConnectedUsers client, (err, users) -> if err? - Utils.getClientAttributes client, ["project_id", "user_id", "doc_id"], (_, {project_id, user_id, doc_id}) -> - logger.error {err, client_id: client.id, user_id, project_id, doc_id}, "server side error in getConnectedUsers" - # Don't return raw error to prevent leaking server side info - return callback {message: "Something went wrong"} + Router._handleError callback, err, client, "clientTracking.getConnectedUsers" else callback(null, users) - \ No newline at end of file + + client.on "clientTracking.updatePosition", (cursorData, callback = (error) ->) -> + WebsocketController.updateClientPosition client, cursorData, (err) -> + if err? + Router._handleError callback, err, client, "clientTracking.updatePosition" + else + callback() \ No newline at end of file diff --git a/services/real-time/app/coffee/WebsocketController.coffee b/services/real-time/app/coffee/WebsocketController.coffee index 7fa4ca6f01..1e7860daf0 100644 --- a/services/real-time/app/coffee/WebsocketController.coffee +++ b/services/real-time/app/coffee/WebsocketController.coffee @@ -70,7 +70,37 @@ module.exports = WebsocketController = client.leave doc_id callback() + updateClientPosition: (client, cursorData, callback = (error) ->) -> + Utils.getClientAttributes client, [ + "project_id", "first_name", "last_name", "email", "user_id" + ], (error, {project_id, first_name, last_name, email, user_id}) -> + return callback(error) if error? + logger.log {user_id, project_id, client_id: client.id, cursorData: cursorData}, "updating client position" + cursorData.id = client.id + cursorData.user_id = user_id if user_id? + cursorData.email = email if email? + if first_name? and last_name? + cursorData.name = first_name + " " + last_name + ConnectedUsersManager.updateUserPosition(project_id, client.id, { + first_name: first_name, + last_name: last_name, + email: email, + user_id: user_id + }, { + row: cursorData.row, + column: cursorData.column, + doc_id: cursorData.doc_id + }, callback) + else + cursorData.name = "Anonymous" + callback() + #EditorRealTimeController.emitToRoom(project_id, "clientTracking.clientUpdated", cursorData) + #callback() + getConnectedUsers: (client, callback = (error, users) ->) -> + Utils.getClientAttributes client, ["project_id", "user_id"], (error, {project_id, user_id}) -> + logger.log {user_id, project_id, client_id: client.id}, "getting connected users" + AuthorizationManager.assertClientCanViewProject client, (error) -> return callback(error) if error? client.get "project_id", (error, project_id) -> diff --git a/services/real-time/test/acceptance/coffee/ClientTrackingTests.coffee b/services/real-time/test/acceptance/coffee/ClientTrackingTests.coffee new file mode 100644 index 0000000000..391030a1c5 --- /dev/null +++ b/services/real-time/test/acceptance/coffee/ClientTrackingTests.coffee @@ -0,0 +1,61 @@ +chai = require("chai") +expect = chai.expect +chai.should() + +RealTimeClient = require "./helpers/RealTimeClient" +MockWebServer = require "./helpers/MockWebServer" +FixturesManager = require "./helpers/FixturesManager" + +describe "clientTracking", -> + before (done) -> + FixturesManager.setUpProject { + privilegeLevel: "owner" + project: { + name: "Test Project" + } + }, (error, data) => + throw error if error? + {@user_id, @project_id} = data + @clientA = RealTimeClient.connect() + @clientB = RealTimeClient.connect() + @clientA.emit "joinProject", { + project_id: @project_id + }, (error) => + throw error if error? + @clientB.emit "joinProject", { + project_id: @project_id + }, (error) => + throw error if error? + done() + + describe "when a client updates its cursor location", -> + before (done) -> + @updates = [] + @clientB.on "clientTracking.clientUpdated", (data) -> + @updates.push data + + @clientA.emit "clientTracking.updatePosition", { + row: @row = 42 + column: @column = 36 + doc_id: @doc_id = "mock-doc-id" + }, (error) -> + throw error if error? + done() + + it "should tell other clients about the update" + + it "should record the update in getConnectedUsers", (done) -> + @clientB.emit "clientTracking.getConnectedUsers", (error, users) => + for user in users + if user.client_id == @clientA.socket.sessionid + expect(user.cursorData).to.deep.equal({ + row: @row + column: @column + doc_id: @doc_id + }) + return done() + throw new Error("user was never found") + + + describe "anonymous users", -> + it "should test something..." diff --git a/services/real-time/test/acceptance/coffee/JoinProjectTests.coffee b/services/real-time/test/acceptance/coffee/JoinProjectTests.coffee index 48b8732080..63633dec24 100644 --- a/services/real-time/test/acceptance/coffee/JoinProjectTests.coffee +++ b/services/real-time/test/acceptance/coffee/JoinProjectTests.coffee @@ -47,7 +47,7 @@ describe "joinProject", -> done() it "should have marked the user as connected", (done) -> - @client.emit "getConnectedUsers", (error, users) => + @client.emit "clientTracking.getConnectedUsers", (error, users) => connected = false for user in users if user.client_id == @client.socket.sessionid and user.user_id == @user_id diff --git a/services/real-time/test/acceptance/coffee/helpers/FixturesManager.coffee b/services/real-time/test/acceptance/coffee/helpers/FixturesManager.coffee index 5c2eb5fd3a..ad584ce608 100644 --- a/services/real-time/test/acceptance/coffee/helpers/FixturesManager.coffee +++ b/services/real-time/test/acceptance/coffee/helpers/FixturesManager.coffee @@ -16,7 +16,11 @@ module.exports = FixturesManager = MockWebServer.run (error) => throw error if error? RealTimeClient.setSession { - user: { _id: user_id } + user: { + _id: user_id + first_name: "Joe" + last_name: "Bloggs" + } }, (error) => throw error if error? callback null, {project_id, user_id, privilegeLevel, project} diff --git a/services/real-time/test/unit/coffee/WebsocketControllerTests.coffee b/services/real-time/test/unit/coffee/WebsocketControllerTests.coffee index 976e113291..8aedca2366 100644 --- a/services/real-time/test/unit/coffee/WebsocketControllerTests.coffee +++ b/services/real-time/test/unit/coffee/WebsocketControllerTests.coffee @@ -217,4 +217,72 @@ describe 'WebsocketController', -> it "should return an error", -> @callback.calledWith(@err).should.equal true - \ No newline at end of file + describe "updateClientPosition", -> + beforeEach -> + #@EditorRealTimeController.emitToRoom = sinon.stub() + @ConnectedUsersManager.updateUserPosition = sinon.stub().callsArgWith(4) + @update = { + doc_id: @doc_id = "doc-id-123" + row: @row = 42 + column: @column = 37 + } + + describe "with a logged in user", -> + beforeEach -> + @clientParams = { + project_id: @project_id + first_name: @first_name = "Douglas" + last_name: @last_name = "Adams" + email: @email = "joe@example.com" + user_id: @user_id = "user-id-123" + } + @client.get = (param, callback) => callback null, @clientParams[param] + @WebsocketController.updateClientPosition @client, @update + + @populatedCursorData = + doc_id: @doc_id, + id: @client.id + name: "#{@first_name} #{@last_name}" + row: @row + column: @column + email: @email + user_id: @user_id + + # it "should send the update to the project room with the user's name", -> + # @EditorRealTimeController.emitToRoom.calledWith(@project_id, "clientTracking.clientUpdated", @populatedCursorData).should.equal true + + it "should send the cursor data to the connected user manager", (done)-> + @ConnectedUsersManager.updateUserPosition.calledWith(@project_id, @client.id, { + user_id: @user_id, + email: @email, + first_name: @first_name, + last_name: @last_name + }, { + row: @row + column: @column + doc_id: @doc_id + }).should.equal true + done() + + describe "with an anonymous user", -> + beforeEach -> + @clientParams = { + project_id: @project_id + } + @client.get = (param, callback) => callback null, @clientParams[param] + @WebsocketController.updateClientPosition @client, @update + + # it "should send the update to the project room with an anonymous name", -> + # @EditorRealTimeController.emitToRoom + # .calledWith(@project_id, "clientTracking.clientUpdated", { + # doc_id: @doc_id, + # id: @client.id + # name: "Anonymous" + # row: @row + # column: @column + # }) + # .should.equal true + + it "should not send cursor data to the connected user manager", (done)-> + @ConnectedUsersManager.updateUserPosition.called.should.equal false + done() \ No newline at end of file From cc1c85ebf88ac643b01e4b61374a07d504cfd905 Mon Sep 17 00:00:00 2001 From: James Allen Date: Thu, 13 Nov 2014 16:03:37 +0000 Subject: [PATCH 013/491] Distribute server side socket.io updates over Redis Pub/Sub --- services/real-time/app.coffee | 3 + .../app/coffee/ConnectedUsersManager.coffee | 1 - .../app/coffee/WebsocketController.coffee | 4 +- .../app/coffee/WebsocketLoadBalancer.coffee | 30 +++++++ .../coffee/ClientTrackingTests.coffee | 14 ++- .../coffee/WebsocketControllerTests.coffee | 27 +++--- .../coffee/WebsocketLoadBalancerTests.coffee | 89 +++++++++++++++++++ 7 files changed, 150 insertions(+), 18 deletions(-) create mode 100644 services/real-time/app/coffee/WebsocketLoadBalancer.coffee create mode 100644 services/real-time/test/unit/coffee/WebsocketLoadBalancerTests.coffee diff --git a/services/real-time/app.coffee b/services/real-time/app.coffee index 15097f51b8..016aa73b5b 100644 --- a/services/real-time/app.coffee +++ b/services/real-time/app.coffee @@ -41,6 +41,9 @@ io.configure -> Router = require "./app/js/Router" Router.configure(app, io, sessionSockets) + +WebsocketLoadBalancer = require "./app/js/WebsocketLoadBalancer" +WebsocketLoadBalancer.listenForEditorEvents(io) port = Settings.internal.realTime.port host = Settings.internal.realTime.host diff --git a/services/real-time/app/coffee/ConnectedUsersManager.coffee b/services/real-time/app/coffee/ConnectedUsersManager.coffee index 5a77660f55..fa7e366ad8 100644 --- a/services/real-time/app/coffee/ConnectedUsersManager.coffee +++ b/services/real-time/app/coffee/ConnectedUsersManager.coffee @@ -61,7 +61,6 @@ module.exports = else result.connected = true result.client_id = client_id - console.log "RESULT", result if result.cursorData? try result.cursorData = JSON.parse(result.cursorData) diff --git a/services/real-time/app/coffee/WebsocketController.coffee b/services/real-time/app/coffee/WebsocketController.coffee index 1e7860daf0..a115023ea5 100644 --- a/services/real-time/app/coffee/WebsocketController.coffee +++ b/services/real-time/app/coffee/WebsocketController.coffee @@ -3,6 +3,7 @@ WebApiManager = require "./WebApiManager" AuthorizationManager = require "./AuthorizationManager" DocumentUpdaterManager = require "./DocumentUpdaterManager" ConnectedUsersManager = require "./ConnectedUsersManager" +WebsocketLoadBalancer = require "./WebsocketLoadBalancer" Utils = require "./Utils" module.exports = WebsocketController = @@ -94,8 +95,7 @@ module.exports = WebsocketController = else cursorData.name = "Anonymous" callback() - #EditorRealTimeController.emitToRoom(project_id, "clientTracking.clientUpdated", cursorData) - #callback() + WebsocketLoadBalancer.emitToRoom(project_id, "clientTracking.clientUpdated", cursorData) getConnectedUsers: (client, callback = (error, users) ->) -> Utils.getClientAttributes client, ["project_id", "user_id"], (error, {project_id, user_id}) -> diff --git a/services/real-time/app/coffee/WebsocketLoadBalancer.coffee b/services/real-time/app/coffee/WebsocketLoadBalancer.coffee new file mode 100644 index 0000000000..966955f117 --- /dev/null +++ b/services/real-time/app/coffee/WebsocketLoadBalancer.coffee @@ -0,0 +1,30 @@ +Settings = require 'settings-sharelatex' +redis = require("redis-sharelatex") +rclientPub = redis.createClient(Settings.redis.web) +rclientSub = redis.createClient(Settings.redis.web) + +module.exports = WebsocketLoadBalancer = + rclientPub: rclientPub + rclientSub: rclientSub + + emitToRoom: (room_id, message, payload...) -> + @rclientPub.publish "editor-events", JSON.stringify + room_id: room_id + message: message + payload: payload + + emitToAll: (message, payload...) -> + @emitToRoom "all", message, payload... + + listenForEditorEvents: (io) -> + @rclientSub.subscribe "editor-events" + @rclientSub.on "message", (channel, message) -> + WebsocketLoadBalancer._processEditorEvent io, channel, message + + _processEditorEvent: (io, channel, message) -> + message = JSON.parse(message) + if message.room_id == "all" + io.sockets.emit(message.message, message.payload...) + else + io.sockets.in(message.room_id).emit(message.message, message.payload...) + diff --git a/services/real-time/test/acceptance/coffee/ClientTrackingTests.coffee b/services/real-time/test/acceptance/coffee/ClientTrackingTests.coffee index 391030a1c5..f83659c5f7 100644 --- a/services/real-time/test/acceptance/coffee/ClientTrackingTests.coffee +++ b/services/real-time/test/acceptance/coffee/ClientTrackingTests.coffee @@ -31,7 +31,7 @@ describe "clientTracking", -> describe "when a client updates its cursor location", -> before (done) -> @updates = [] - @clientB.on "clientTracking.clientUpdated", (data) -> + @clientB.on "clientTracking.clientUpdated", (data) => @updates.push data @clientA.emit "clientTracking.updatePosition", { @@ -42,7 +42,17 @@ describe "clientTracking", -> throw error if error? done() - it "should tell other clients about the update" + it "should tell other clients about the update", -> + @updates.should.deep.equal [ + { + row: @row + column: @column + doc_id: @doc_id + id: @clientA.socket.sessionid + user_id: @user_id + name: "Joe Bloggs" + } + ] it "should record the update in getConnectedUsers", (done) -> @clientB.emit "clientTracking.getConnectedUsers", (error, users) => diff --git a/services/real-time/test/unit/coffee/WebsocketControllerTests.coffee b/services/real-time/test/unit/coffee/WebsocketControllerTests.coffee index 8aedca2366..e0be68fc50 100644 --- a/services/real-time/test/unit/coffee/WebsocketControllerTests.coffee +++ b/services/real-time/test/unit/coffee/WebsocketControllerTests.coffee @@ -29,6 +29,7 @@ describe 'WebsocketController', -> "./AuthorizationManager": @AuthorizationManager = {} "./DocumentUpdaterManager": @DocumentUpdaterManager = {} "./ConnectedUsersManager": @ConnectedUsersManager = {} + "./WebsocketLoadBalancer": @WebsocketLoadBalancer = {} "logger-sharelatex": @logger = { log: sinon.stub(), error: sinon.stub() } afterEach -> @@ -219,7 +220,7 @@ describe 'WebsocketController', -> describe "updateClientPosition", -> beforeEach -> - #@EditorRealTimeController.emitToRoom = sinon.stub() + @WebsocketLoadBalancer.emitToRoom = sinon.stub() @ConnectedUsersManager.updateUserPosition = sinon.stub().callsArgWith(4) @update = { doc_id: @doc_id = "doc-id-123" @@ -248,8 +249,8 @@ describe 'WebsocketController', -> email: @email user_id: @user_id - # it "should send the update to the project room with the user's name", -> - # @EditorRealTimeController.emitToRoom.calledWith(@project_id, "clientTracking.clientUpdated", @populatedCursorData).should.equal true + it "should send the update to the project room with the user's name", -> + @WebsocketLoadBalancer.emitToRoom.calledWith(@project_id, "clientTracking.clientUpdated", @populatedCursorData).should.equal true it "should send the cursor data to the connected user manager", (done)-> @ConnectedUsersManager.updateUserPosition.calledWith(@project_id, @client.id, { @@ -272,16 +273,16 @@ describe 'WebsocketController', -> @client.get = (param, callback) => callback null, @clientParams[param] @WebsocketController.updateClientPosition @client, @update - # it "should send the update to the project room with an anonymous name", -> - # @EditorRealTimeController.emitToRoom - # .calledWith(@project_id, "clientTracking.clientUpdated", { - # doc_id: @doc_id, - # id: @client.id - # name: "Anonymous" - # row: @row - # column: @column - # }) - # .should.equal true + it "should send the update to the project room with an anonymous name", -> + @WebsocketLoadBalancer.emitToRoom + .calledWith(@project_id, "clientTracking.clientUpdated", { + doc_id: @doc_id, + id: @client.id + name: "Anonymous" + row: @row + column: @column + }) + .should.equal true it "should not send cursor data to the connected user manager", (done)-> @ConnectedUsersManager.updateUserPosition.called.should.equal false diff --git a/services/real-time/test/unit/coffee/WebsocketLoadBalancerTests.coffee b/services/real-time/test/unit/coffee/WebsocketLoadBalancerTests.coffee new file mode 100644 index 0000000000..547d0aff58 --- /dev/null +++ b/services/real-time/test/unit/coffee/WebsocketLoadBalancerTests.coffee @@ -0,0 +1,89 @@ +SandboxedModule = require('sandboxed-module') +sinon = require('sinon') +require('chai').should() +modulePath = require('path').join __dirname, '../../../app/js/WebsocketLoadBalancer' + +describe "WebsocketLoadBalancer", -> + beforeEach -> + @WebsocketLoadBalancer = SandboxedModule.require modulePath, requires: + "redis-sharelatex": + createClient: () -> + auth:-> + @io = {} + @WebsocketLoadBalancer.rclientPub = publish: sinon.stub() + @WebsocketLoadBalancer.rclientSub = + subscribe: sinon.stub() + on: sinon.stub() + + @room_id = "room-id" + @message = "message-to-editor" + @payload = ["argument one", 42] + + describe "emitToRoom", -> + beforeEach -> + @WebsocketLoadBalancer.emitToRoom(@room_id, @message, @payload...) + + it "should publish the message to redis", -> + @WebsocketLoadBalancer.rclientPub.publish + .calledWith("editor-events", JSON.stringify( + room_id: @room_id, + message: @message + payload: @payload + )) + .should.equal true + + describe "emitToAll", -> + beforeEach -> + @WebsocketLoadBalancer.emitToRoom = sinon.stub() + @WebsocketLoadBalancer.emitToAll @message, @payload... + + it "should emit to the room 'all'", -> + @WebsocketLoadBalancer.emitToRoom + .calledWith("all", @message, @payload...) + .should.equal true + + describe "listenForEditorEvents", -> + beforeEach -> + @WebsocketLoadBalancer._processEditorEvent = sinon.stub() + @WebsocketLoadBalancer.listenForEditorEvents() + + it "should subscribe to the editor-events channel", -> + @WebsocketLoadBalancer.rclientSub.subscribe + .calledWith("editor-events") + .should.equal true + + it "should process the events with _processEditorEvent", -> + @WebsocketLoadBalancer.rclientSub.on + .calledWith("message", sinon.match.func) + .should.equal true + + describe "_processEditorEvent", -> + describe "with a designated room", -> + beforeEach -> + @io.sockets = + in: sinon.stub().returns(emit: @emit = sinon.stub()) + data = JSON.stringify + room_id: @room_id + message: @message + payload: @payload + @WebsocketLoadBalancer._processEditorEvent(@io, "editor-events", data) + + it "should send the message to all clients in the room", -> + @io.sockets.in + .calledWith(@room_id) + .should.equal true + @emit.calledWith(@message, @payload...).should.equal true + + describe "when emitting to all", -> + beforeEach -> + @io.sockets = + emit: @emit = sinon.stub() + data = JSON.stringify + room_id: "all" + message: @message + payload: @payload + @WebsocketLoadBalancer._processEditorEvent(@io, "editor-events", data) + + it "should send the message to all clients", -> + @emit.calledWith(@message, @payload...).should.equal true + From f0e69bfe2d4c3877d27048f458906026677aece1 Mon Sep 17 00:00:00 2001 From: James Allen Date: Thu, 13 Nov 2014 17:07:05 +0000 Subject: [PATCH 014/491] Add appltOtUpdate end point (sans acceptance tests for now) --- .../app/coffee/AuthorizationManager.coffee | 5 +- .../app/coffee/DocumentUpdaterManager.coffee | 16 ++++- .../app/coffee/WebsocketController.coffee | 24 +++++++ .../acceptance/coffee/ApplyUpdateTests.coffee | 12 ++++ .../coffee/AuthorizationManagerTests.coffee | 25 +++++++ .../coffee/DocumentUpdaterManagerTests.coffee | 43 ++++++++++++ .../coffee/WebsocketControllerTests.coffee | 70 ++++++++++++++++++- 7 files changed, 191 insertions(+), 4 deletions(-) create mode 100644 services/real-time/test/acceptance/coffee/ApplyUpdateTests.coffee diff --git a/services/real-time/app/coffee/AuthorizationManager.coffee b/services/real-time/app/coffee/AuthorizationManager.coffee index 345bfa4804..273e9697cf 100644 --- a/services/real-time/app/coffee/AuthorizationManager.coffee +++ b/services/real-time/app/coffee/AuthorizationManager.coffee @@ -1,7 +1,10 @@ module.exports = AuthorizationManager = assertClientCanViewProject: (client, callback = (error) ->) -> AuthorizationManager._assertClientHasPrivilegeLevel client, ["readOnly", "readAndWrite", "owner"], callback - + + assertClientCanEditProject: (client, callback = (error) ->) -> + AuthorizationManager._assertClientHasPrivilegeLevel client, ["readAndWrite", "owner"], callback + _assertClientHasPrivilegeLevel: (client, allowedLevels, callback = (error) ->) -> client.get "privilege_level", (error, privilegeLevel) -> return callback(error) if error? diff --git a/services/real-time/app/coffee/DocumentUpdaterManager.coffee b/services/real-time/app/coffee/DocumentUpdaterManager.coffee index 54ba6f9072..2af68eef0c 100644 --- a/services/real-time/app/coffee/DocumentUpdaterManager.coffee +++ b/services/real-time/app/coffee/DocumentUpdaterManager.coffee @@ -2,6 +2,9 @@ request = require "request" logger = require "logger-sharelatex" settings = require "settings-sharelatex" +redis = require("redis-sharelatex") +rclient = redis.createClient(settings.redis.web) + module.exports = DocumentUpdaterManager = getDocument: (project_id, doc_id, fromVersion, callback = (error, exists, doclines, version) ->) -> #timer = new metrics.Timer("get-document") @@ -23,4 +26,15 @@ module.exports = DocumentUpdaterManager = err = new Error("doc updater returned a non-success status code: #{res.statusCode}") err.statusCode = res.statusCode logger.error {err, project_id, doc_id, url}, "doc updater returned a non-success status code: #{res.statusCode}" - callback err \ No newline at end of file + callback err + + queueChange: (project_id, doc_id, change, callback = ()->)-> + jsonChange = JSON.stringify change + doc_key = "#{project_id}:#{doc_id}" + multi = rclient.multi() + multi.rpush "PendingUpdates:#{doc_id}", jsonChange + multi.sadd "DocsWithPendingUpdates", doc_key + multi.rpush "pending-updates-list", doc_key + multi.exec (error) -> + return callback(error) if error? + callback() \ No newline at end of file diff --git a/services/real-time/app/coffee/WebsocketController.coffee b/services/real-time/app/coffee/WebsocketController.coffee index a115023ea5..0bdc0fa019 100644 --- a/services/real-time/app/coffee/WebsocketController.coffee +++ b/services/real-time/app/coffee/WebsocketController.coffee @@ -109,3 +109,27 @@ module.exports = WebsocketController = ConnectedUsersManager.getConnectedUsers project_id, (error, users) -> return callback(error) if error? callback null, users + + applyOtUpdate: (client, project_id, doc_id, update, callback = (error) ->) -> + AuthorizationManager.assertClientCanEditProject client, (error) -> + if error? + logger.error {err: error, project_id, doc_id, client_id: client.id, version: update.v}, "client is not authorized to make update" + client.disconnect() + return callback(error) + + Utils.getClientAttributes client, ["user_id"], (error, {user_id}) -> + return callback(error) if error? + update.meta ||= {} + update.meta.source = client.id + update.meta.user_id = user_id + # metrics.inc "editor.doc-update", 0.3 + # metrics.set "editor.active-projects", project_id, 0.3 + # metrics.set "editor.active-users", user_id, 0.3 + + logger.log {user_id, doc_id, project_id, client_id: client.id, version: update.v}, "sending update to doc updater" + + DocumentUpdaterManager.queueChange project_id, doc_id, update, (error) -> + if error? + logger.error {err: error, project_id, doc_id, client_id: client.id, version: update.v}, "document was not available for update" + client.disconnect() + callback(error) diff --git a/services/real-time/test/acceptance/coffee/ApplyUpdateTests.coffee b/services/real-time/test/acceptance/coffee/ApplyUpdateTests.coffee new file mode 100644 index 0000000000..284117b4d1 --- /dev/null +++ b/services/real-time/test/acceptance/coffee/ApplyUpdateTests.coffee @@ -0,0 +1,12 @@ +describe "applyOtUpdate", -> + describe "when authorized", -> + it "should publish a message on redis" + + it "should add the doc to the pending updates set in redis" + + it "should push the update into redis" + + describe "when not authorized", -> + it "should return an error" + + it "should disconnect the client" \ No newline at end of file diff --git a/services/real-time/test/unit/coffee/AuthorizationManagerTests.coffee b/services/real-time/test/unit/coffee/AuthorizationManagerTests.coffee index cd3822538b..a4741ab4d8 100644 --- a/services/real-time/test/unit/coffee/AuthorizationManagerTests.coffee +++ b/services/real-time/test/unit/coffee/AuthorizationManagerTests.coffee @@ -35,5 +35,30 @@ describe 'AuthorizationManager', -> it "should return an error with any other privilegeLevel", (done) -> @client.params.privilege_level = "unknown" @AuthorizationManager.assertClientCanViewProject @client, (error) -> + error.message.should.equal "not authorized" + done() + + describe "assertClientCanEditProject", -> + it "should not allow the readOnly privilegeLevel", (done) -> + @client.params.privilege_level = "readOnly" + @AuthorizationManager.assertClientCanEditProject @client, (error) -> + error.message.should.equal "not authorized" + done() + + it "should allow the readAndWrite privilegeLevel", (done) -> + @client.params.privilege_level = "readAndWrite" + @AuthorizationManager.assertClientCanEditProject @client, (error) -> + expect(error).to.be.null + done() + + it "should allow the owner privilegeLevel", (done) -> + @client.params.privilege_level = "owner" + @AuthorizationManager.assertClientCanEditProject @client, (error) -> + expect(error).to.be.null + done() + + it "should return an error with any other privilegeLevel", (done) -> + @client.params.privilege_level = "unknown" + @AuthorizationManager.assertClientCanEditProject @client, (error) -> error.message.should.equal "not authorized" done() \ No newline at end of file diff --git a/services/real-time/test/unit/coffee/DocumentUpdaterManagerTests.coffee b/services/real-time/test/unit/coffee/DocumentUpdaterManagerTests.coffee index 9f80981374..80caa45802 100644 --- a/services/real-time/test/unit/coffee/DocumentUpdaterManagerTests.coffee +++ b/services/real-time/test/unit/coffee/DocumentUpdaterManagerTests.coffee @@ -12,11 +12,14 @@ describe 'DocumentUpdaterManager', -> @version = 42 @settings = apis: documentupdater: url: "http://doc-updater.example.com" + redis: web: {} + @rclient = {auth:->} @DocumentUpdaterManager = SandboxedModule.require modulePath, requires: 'settings-sharelatex':@settings 'logger-sharelatex': @logger = {log: sinon.stub(), error: sinon.stub()} 'request': @request = {} + 'redis-sharelatex' : createClient: () => @rclient describe "getDocument", -> beforeEach -> @@ -58,3 +61,43 @@ describe 'DocumentUpdaterManager', -> @callback .calledWith(err) .should.equal true + + describe 'queueChange', -> + beforeEach -> + @change = { + "action":"removeText", + "range":{"start":{"row":2,"column":2},"end":{"row":2,"column":3}}, + "text":"e" + } + @rclient.multi = sinon.stub().returns @rclient + @rclient.exec = sinon.stub().callsArg(0) + @rclient.rpush = sinon.stub() + @rclient.sadd = sinon.stub() + @callback = sinon.stub() + + describe "successfully", -> + beforeEach -> + @DocumentUpdaterManager.queueChange(@project_id, @doc_id, @change, @callback) + + it "should push the change", -> + @rclient.rpush + .calledWith("PendingUpdates:#{@doc_id}", JSON.stringify(@change)) + .should.equal true + + it "should notify the doc updater of the change via the pending-updates-list queue", -> + @rclient.rpush + .calledWith("pending-updates-list", "#{@project_id}:#{@doc_id}") + .should.equal true + + it "should push the doc id into the pending updates set", -> + @rclient.sadd + .calledWith("DocsWithPendingUpdates", "#{@project_id}:#{@doc_id}") + .should.equal true + + describe "with error connecting to redis during exec", -> + beforeEach -> + @rclient.exec = sinon.stub().callsArgWith(0, new Error("something went wrong")) + @DocumentUpdaterManager.queueChange(@project_id, @doc_id, @change, @callback) + + it "should return an error", -> + @callback.calledWithExactly(sinon.match(Error)).should.equal true diff --git a/services/real-time/test/unit/coffee/WebsocketControllerTests.coffee b/services/real-time/test/unit/coffee/WebsocketControllerTests.coffee index e0be68fc50..3e2a73b6c3 100644 --- a/services/real-time/test/unit/coffee/WebsocketControllerTests.coffee +++ b/services/real-time/test/unit/coffee/WebsocketControllerTests.coffee @@ -10,7 +10,7 @@ describe 'WebsocketController', -> tk.freeze(new Date()) @project_id = "project-id-123" @user = { - _id: "user-id-123" + _id: @user_id = "user-id-123" first_name: "James" last_name: "Allen" email: "james@example.com" @@ -19,6 +19,7 @@ describe 'WebsocketController', -> } @callback = sinon.stub() @client = + id: @client_id = "mock-client-id-123" params: {} set: sinon.stub() get: (param, cb) -> cb null, @params[param] @@ -286,4 +287,69 @@ describe 'WebsocketController', -> it "should not send cursor data to the connected user manager", (done)-> @ConnectedUsersManager.updateUserPosition.called.should.equal false - done() \ No newline at end of file + done() + + describe "applyOtUpdate", -> + beforeEach -> + @update = {op: {p: 12, t: "foo"}} + @client.params.user_id = @user_id + @AuthorizationManager.assertClientCanEditProject = sinon.stub().callsArg(1) + @DocumentUpdaterManager.queueChange = sinon.stub().callsArg(3) + + describe "succesfully", -> + beforeEach -> + @WebsocketController.applyOtUpdate @client, @project_id, @doc_id, @update, @callback + + it "should set the source of the update to the client id", -> + @update.meta.source.should.equal @client.id + + it "should set the user_id of the update to the user id", -> + @update.meta.user_id.should.equal @user_id + + it "should queue the update", -> + @DocumentUpdaterManager.queueChange + .calledWith(@project_id, @doc_id, @update) + .should.equal true + + it "should call the callback", -> + @callback.called.should.equal true + + # it "should update the active users metric", -> + # @metrics.set.calledWith("editor.active-users", @user_id).should.equal true + # + # it "should update the active projects metric", -> + # @metrics.set.calledWith("editor.active-projects", @project_id).should.equal true + # + # it "should increment the doc updates", -> + # @metrics.inc.calledWith("editor.doc-update").should.equal true + + describe "unsuccessfully", -> + beforeEach -> + @client.disconnect = sinon.stub() + @DocumentUpdaterManager.queueChange = sinon.stub().callsArgWith(3, @error = new Error("Something went wrong")) + @WebsocketController.applyOtUpdate @client, @project_id, @doc_id, @update, @callback + + it "should disconnect the client", -> + @client.disconnect.called.should.equal true + + it "should log an error", -> + @logger.error.called.should.equal true + + it "should call the callback with the error", -> + @callback.calledWith(@error).should.equal true + + describe "when not authorized", -> + beforeEach -> + @client.disconnect = sinon.stub() + @AuthorizationManager.assertClientCanEditProject = sinon.stub().callsArgWith(1, @error = new Error("not authorized")) + @WebsocketController.applyOtUpdate @client, @project_id, @doc_id, @update, @callback + + it "should disconnect the client", -> + @client.disconnect.called.should.equal true + + it "should log an error", -> + @logger.error.called.should.equal true + + it "should call the callback with the error", -> + @callback.calledWith(@error).should.equal true + \ No newline at end of file From fef5f6b775de4f68bed3e4e8bc3698967bfb4e84 Mon Sep 17 00:00:00 2001 From: James Allen Date: Fri, 14 Nov 2014 10:12:35 +0000 Subject: [PATCH 015/491] Add acceptance tests for applyOtUpdate --- services/real-time/app/coffee/Router.coffee | 8 ++ .../app/coffee/WebsocketController.coffee | 11 +- .../acceptance/coffee/ApplyUpdateTests.coffee | 104 +++++++++++++++++- .../coffee/WebsocketControllerTests.coffee | 7 +- 4 files changed, 118 insertions(+), 12 deletions(-) diff --git a/services/real-time/app/coffee/Router.coffee b/services/real-time/app/coffee/Router.coffee index 43ec7976f0..7872727377 100644 --- a/services/real-time/app/coffee/Router.coffee +++ b/services/real-time/app/coffee/Router.coffee @@ -13,6 +13,7 @@ module.exports = Router = attrs.err = error logger.error attrs, "server side error in #{method}" # Don't return raw error to prevent leaking server side info + console.log "CALLING CALLBACK", callback return callback {message: "Something went wrong"} configure: (app, io, session) -> @@ -74,5 +75,12 @@ module.exports = Router = WebsocketController.updateClientPosition client, cursorData, (err) -> if err? Router._handleError callback, err, client, "clientTracking.updatePosition" + else + callback() + + client.on "applyOtUpdate", (doc_id, update, callback = (error) ->) -> + WebsocketController.applyOtUpdate client, doc_id, update, (err) -> + if err? + Router._handleError callback, err, client, "applyOtUpdate", {doc_id, update} else callback() \ No newline at end of file diff --git a/services/real-time/app/coffee/WebsocketController.coffee b/services/real-time/app/coffee/WebsocketController.coffee index 0bdc0fa019..6f1b3b5119 100644 --- a/services/real-time/app/coffee/WebsocketController.coffee +++ b/services/real-time/app/coffee/WebsocketController.coffee @@ -110,14 +110,17 @@ module.exports = WebsocketController = return callback(error) if error? callback null, users - applyOtUpdate: (client, project_id, doc_id, update, callback = (error) ->) -> + applyOtUpdate: (client, doc_id, update, callback = (error) ->) -> AuthorizationManager.assertClientCanEditProject client, (error) -> if error? - logger.error {err: error, project_id, doc_id, client_id: client.id, version: update.v}, "client is not authorized to make update" - client.disconnect() + logger.error {err: error, doc_id, client_id: client.id, version: update.v}, "client is not authorized to make update" + setTimeout () -> + # Disconnect, but give the client the chance to receive the error + client.disconnect() + , 100 return callback(error) - Utils.getClientAttributes client, ["user_id"], (error, {user_id}) -> + Utils.getClientAttributes client, ["user_id", "project_id"], (error, {user_id, project_id}) -> return callback(error) if error? update.meta ||= {} update.meta.source = client.id diff --git a/services/real-time/test/acceptance/coffee/ApplyUpdateTests.coffee b/services/real-time/test/acceptance/coffee/ApplyUpdateTests.coffee index 284117b4d1..b5676909cb 100644 --- a/services/real-time/test/acceptance/coffee/ApplyUpdateTests.coffee +++ b/services/real-time/test/acceptance/coffee/ApplyUpdateTests.coffee @@ -1,12 +1,106 @@ +async = require "async" +chai = require("chai") +expect = chai.expect +chai.should() + +RealTimeClient = require "./helpers/RealTimeClient" +FixturesManager = require "./helpers/FixturesManager" + +settings = require "settings-sharelatex" +redis = require "redis-sharelatex" +rclient = redis.createClient(settings.redis.web) + describe "applyOtUpdate", -> + before -> + @update = { + op: [{i: "foo", p: 42}] + } describe "when authorized", -> - it "should publish a message on redis" + before (done) -> + async.series [ + (cb) => + FixturesManager.setUpProject { + privilegeLevel: "readAndWrite" + }, (e, {@project_id, @user_id}) => + cb(e) + + (cb) => + FixturesManager.setUpDoc @project_id, {@lines, @version, @ops}, (e, {@doc_id}) => + cb(e) + + (cb) => + @client = RealTimeClient.connect() + @client.emit "joinProject", project_id: @project_id, cb + + (cb) => + @client.emit "joinDoc", @doc_id, cb + + (cb) => + @client.emit "applyOtUpdate", @doc_id, @update, cb + ], done - it "should add the doc to the pending updates set in redis" + it "should push the doc into the pending updates list", (done) -> + rclient.lrange "pending-updates-list", 0, -1, (error, [doc_id]) => + doc_id.should.equal "#{@project_id}:#{@doc_id}" + done() - it "should push the update into redis" + it "should add the doc to the pending updates set in redis", (done) -> + rclient.sismember "DocsWithPendingUpdates", "#{@project_id}:#{@doc_id}", (error, isMember) => + isMember.should.equal 1 + done() + + it "should push the update into redis", (done) -> + rclient.lrange "PendingUpdates:#{@doc_id}", 0, -1, (error, [update]) => + update = JSON.parse(update) + update.op.should.deep.equal @update.op + update.meta.should.deep.equal { + source: @client.socket.sessionid + user_id: @user_id + } + done() + + after (done) -> + async.series [ + (cb) => rclient.del "pending-updates-list", cb + (cb) => rclient.del "DocsWithPendingUpdates", "#{@project_id}:#{@doc_id}", cb + (cb) => rclient.del "PendingUpdates:#{@doc_id}", cb + ], done describe "when not authorized", -> - it "should return an error" + before (done) -> + async.series [ + (cb) => + FixturesManager.setUpProject { + privilegeLevel: "readOnly" + }, (e, {@project_id, @user_id}) => + cb(e) + + (cb) => + FixturesManager.setUpDoc @project_id, {@lines, @version, @ops}, (e, {@doc_id}) => + cb(e) + + (cb) => + @client = RealTimeClient.connect() + @client.emit "joinProject", project_id: @project_id, cb + + (cb) => + @client.emit "joinDoc", @doc_id, cb + + (cb) => + @client.emit "applyOtUpdate", @doc_id, @update, (@error) => + cb() + ], done - it "should disconnect the client" \ No newline at end of file + it "should return an error", -> + expect(@error).to.exist + + it "should disconnect the client", (done) -> + setTimeout () => + @client.socket.connected.should.equal false + done() + , 300 + + it "should not put the update in redis", (done) -> + rclient.llen "PendingUpdates:#{@doc_id}", (error, len) => + len.should.equal 0 + done() \ No newline at end of file diff --git a/services/real-time/test/unit/coffee/WebsocketControllerTests.coffee b/services/real-time/test/unit/coffee/WebsocketControllerTests.coffee index 3e2a73b6c3..8ea62f466b 100644 --- a/services/real-time/test/unit/coffee/WebsocketControllerTests.coffee +++ b/services/real-time/test/unit/coffee/WebsocketControllerTests.coffee @@ -293,12 +293,13 @@ describe 'WebsocketController', -> beforeEach -> @update = {op: {p: 12, t: "foo"}} @client.params.user_id = @user_id + @client.params.project_id = @project_id @AuthorizationManager.assertClientCanEditProject = sinon.stub().callsArg(1) @DocumentUpdaterManager.queueChange = sinon.stub().callsArg(3) describe "succesfully", -> beforeEach -> - @WebsocketController.applyOtUpdate @client, @project_id, @doc_id, @update, @callback + @WebsocketController.applyOtUpdate @client, @doc_id, @update, @callback it "should set the source of the update to the client id", -> @update.meta.source.should.equal @client.id @@ -327,7 +328,7 @@ describe 'WebsocketController', -> beforeEach -> @client.disconnect = sinon.stub() @DocumentUpdaterManager.queueChange = sinon.stub().callsArgWith(3, @error = new Error("Something went wrong")) - @WebsocketController.applyOtUpdate @client, @project_id, @doc_id, @update, @callback + @WebsocketController.applyOtUpdate @client, @doc_id, @update, @callback it "should disconnect the client", -> @client.disconnect.called.should.equal true @@ -342,7 +343,7 @@ describe 'WebsocketController', -> beforeEach -> @client.disconnect = sinon.stub() @AuthorizationManager.assertClientCanEditProject = sinon.stub().callsArgWith(1, @error = new Error("not authorized")) - @WebsocketController.applyOtUpdate @client, @project_id, @doc_id, @update, @callback + @WebsocketController.applyOtUpdate @client, @doc_id, @update, @callback it "should disconnect the client", -> @client.disconnect.called.should.equal true From b6f51fdafd0209b892dd3617b109d268460d95e4 Mon Sep 17 00:00:00 2001 From: James Allen Date: Fri, 14 Nov 2014 10:21:54 +0000 Subject: [PATCH 016/491] Refactor acceptance tests to wait for connection before proceeding --- .../coffee/ClientTrackingTests.coffee | 42 +++--- .../acceptance/coffee/JoinDocTests.coffee | 134 +++++++++++------- .../acceptance/coffee/JoinProjectTests.coffee | 64 +++++---- .../acceptance/coffee/LeaveDocTests.coffee | 35 +++-- 4 files changed, 166 insertions(+), 109 deletions(-) diff --git a/services/real-time/test/acceptance/coffee/ClientTrackingTests.coffee b/services/real-time/test/acceptance/coffee/ClientTrackingTests.coffee index f83659c5f7..c0f0079bbc 100644 --- a/services/real-time/test/acceptance/coffee/ClientTrackingTests.coffee +++ b/services/real-time/test/acceptance/coffee/ClientTrackingTests.coffee @@ -6,27 +6,35 @@ RealTimeClient = require "./helpers/RealTimeClient" MockWebServer = require "./helpers/MockWebServer" FixturesManager = require "./helpers/FixturesManager" +async = require "async" + describe "clientTracking", -> before (done) -> - FixturesManager.setUpProject { - privilegeLevel: "owner" - project: { - name: "Test Project" - } - }, (error, data) => - throw error if error? - {@user_id, @project_id} = data - @clientA = RealTimeClient.connect() - @clientB = RealTimeClient.connect() - @clientA.emit "joinProject", { - project_id: @project_id - }, (error) => - throw error if error? + async.series [ + (cb) => + FixturesManager.setUpProject { + privilegeLevel: "owner" + project: { name: "Test Project" } + }, (error, {@user_id, @project_id}) => cb() + + (cb) => + @clientA = RealTimeClient.connect() + @clientA.on "connect", cb + + (cb) => + @clientB = RealTimeClient.connect() + @clientB.on "connect", cb + + (cb) => + @clientA.emit "joinProject", { + project_id: @project_id + }, cb + + (cb) => @clientB.emit "joinProject", { project_id: @project_id - }, (error) => - throw error if error? - done() + }, cb + ], done describe "when a client updates its cursor location", -> before (done) -> diff --git a/services/real-time/test/acceptance/coffee/JoinDocTests.coffee b/services/real-time/test/acceptance/coffee/JoinDocTests.coffee index 707108ebf6..ed26ea85cf 100644 --- a/services/real-time/test/acceptance/coffee/JoinDocTests.coffee +++ b/services/real-time/test/acceptance/coffee/JoinDocTests.coffee @@ -6,6 +6,8 @@ RealTimeClient = require "./helpers/RealTimeClient" MockDocUpdaterServer = require "./helpers/MockDocUpdaterServer" FixturesManager = require "./helpers/FixturesManager" +async = require "async" + describe "joinDoc", -> before -> @lines = ["test", "doc", "lines"] @@ -14,20 +16,27 @@ describe "joinDoc", -> describe "when authorised readAndWrite", -> before (done) -> - FixturesManager.setUpProject { - privilegeLevel: "readAndWrite" - }, (error, data) => - throw error if error? - {@project_id, @user_id} = data - FixturesManager.setUpDoc @project_id, {@lines, @version, @ops}, (error, data) => - throw error if error? - {@doc_id} = data + async.series [ + (cb) => + FixturesManager.setUpProject { + privilegeLevel: "readAndWrite" + }, (e, {@project_id, @user_id}) => + cb(e) + + (cb) => + FixturesManager.setUpDoc @project_id, {@lines, @version, @ops}, (e, {@doc_id}) => + cb(e) + + (cb) => @client = RealTimeClient.connect() - @client.emit "joinProject", project_id: @project_id, (error) => - throw error if error? - @client.emit "joinDoc", @doc_id, (error, @returnedArgs...) => - throw error if error? - done() + @client.on "connect", cb + + (cb) => + @client.emit "joinProject", project_id: @project_id, cb + + (cb) => + @client.emit "joinDoc", @doc_id, (error, @returnedArgs...) => cb(error) + ], done it "should get the doc from the doc updater", -> MockDocUpdaterServer.getDocument @@ -44,20 +53,27 @@ describe "joinDoc", -> describe "when authorised readOnly", -> before (done) -> - FixturesManager.setUpProject { - privilegeLevel: "readOnly" - }, (error, data) => - throw error if error? - {@project_id, @user_id} = data - FixturesManager.setUpDoc @project_id, {@lines, @version, @ops}, (error, data) => - throw error if error? - {@doc_id} = data + async.series [ + (cb) => + FixturesManager.setUpProject { + privilegeLevel: "readOnly" + }, (e, {@project_id, @user_id}) => + cb(e) + + (cb) => + FixturesManager.setUpDoc @project_id, {@lines, @version, @ops}, (e, {@doc_id}) => + cb(e) + + (cb) => @client = RealTimeClient.connect() - @client.emit "joinProject", project_id: @project_id, (error) => - throw error if error? - @client.emit "joinDoc", @doc_id, (error, @returnedArgs...) => - throw error if error? - done() + @client.on "connect", cb + + (cb) => + @client.emit "joinProject", project_id: @project_id, cb + + (cb) => + @client.emit "joinDoc", @doc_id, (error, @returnedArgs...) => cb(error) + ], done it "should get the doc from the doc updater", -> MockDocUpdaterServer.getDocument @@ -74,20 +90,27 @@ describe "joinDoc", -> describe "when authorised as owner", -> before (done) -> - FixturesManager.setUpProject { - privilegeLevel: "owner" - }, (error, data) => - throw error if error? - {@project_id, @user_id} = data - FixturesManager.setUpDoc @project_id, {@lines, @version, @ops}, (error, data) => - throw error if error? - {@doc_id} = data + async.series [ + (cb) => + FixturesManager.setUpProject { + privilegeLevel: "owner" + }, (e, {@project_id, @user_id}) => + cb(e) + + (cb) => + FixturesManager.setUpDoc @project_id, {@lines, @version, @ops}, (e, {@doc_id}) => + cb(e) + + (cb) => @client = RealTimeClient.connect() - @client.emit "joinProject", project_id: @project_id, (error) => - throw error if error? - @client.emit "joinDoc", @doc_id, (error, @returnedArgs...) => - throw error if error? - done() + @client.on "connect", cb + + (cb) => + @client.emit "joinProject", project_id: @project_id, cb + + (cb) => + @client.emit "joinDoc", @doc_id, (error, @returnedArgs...) => cb(error) + ], done it "should get the doc from the doc updater", -> MockDocUpdaterServer.getDocument @@ -109,20 +132,27 @@ describe "joinDoc", -> describe "with a fromVersion", -> before (done) -> @fromVersion = 36 - FixturesManager.setUpProject { - privilegeLevel: "readAndWrite" - }, (error, data) => - throw error if error? - {@project_id, @user_id} = data - FixturesManager.setUpDoc @project_id, {@lines, @version, @ops}, (error, data) => - throw error if error? - {@doc_id} = data + async.series [ + (cb) => + FixturesManager.setUpProject { + privilegeLevel: "readAndWrite" + }, (e, {@project_id, @user_id}) => + cb(e) + + (cb) => + FixturesManager.setUpDoc @project_id, {@lines, @version, @ops}, (e, {@doc_id}) => + cb(e) + + (cb) => @client = RealTimeClient.connect() - @client.emit "joinProject", project_id: @project_id, (error) => - throw error if error? - @client.emit "joinDoc", @doc_id, @fromVersion, (error, @returnedArgs...) => - throw error if error? - done() + @client.on "connect", cb + + (cb) => + @client.emit "joinProject", project_id: @project_id, cb + + (cb) => + @client.emit "joinDoc", @doc_id, @fromVersion, (error, @returnedArgs...) => cb(error) + ], done it "should get the doc from the doc updater with the fromVersion", -> MockDocUpdaterServer.getDocument diff --git a/services/real-time/test/acceptance/coffee/JoinProjectTests.coffee b/services/real-time/test/acceptance/coffee/JoinProjectTests.coffee index 63633dec24..920330b395 100644 --- a/services/real-time/test/acceptance/coffee/JoinProjectTests.coffee +++ b/services/real-time/test/acceptance/coffee/JoinProjectTests.coffee @@ -6,24 +6,29 @@ RealTimeClient = require "./helpers/RealTimeClient" MockWebServer = require "./helpers/MockWebServer" FixturesManager = require "./helpers/FixturesManager" +async = require "async" describe "joinProject", -> describe "when authorized", -> before (done) -> - FixturesManager.setUpProject { - privilegeLevel: "owner" - project: { - name: "Test Project" - } - }, (error, data) => - throw error if error? - {@user_id, @project_id} = data - @client = RealTimeClient.connect() - @client.emit "joinProject", { - project_id: @project_id - }, (error, @project, @privilegeLevel, @protocolVersion) => - throw error if error? - done() + async.series [ + (cb) => + FixturesManager.setUpProject { + privilegeLevel: "owner" + project: { + name: "Test Project" + } + }, (e, {@project_id, @user_id}) => + cb(e) + + (cb) => + @client = RealTimeClient.connect() + @client.on "connect", cb + + (cb) => + @client.emit "joinProject", project_id: @project_id, (error, @project, @privilegeLevel, @protocolVersion) => + cb(error) + ], done it "should get the project from web", -> MockWebServer.joinProject @@ -58,19 +63,24 @@ describe "joinProject", -> describe "when not authorized", -> before (done) -> - FixturesManager.setUpProject { - privilegeLevel: null - project: { - name: "Test Project" - } - }, (error, data) => - throw error if error? - {@user_id, @project_id} = data - @client = RealTimeClient.connect() - @client.emit "joinProject", { - project_id: @project_id - }, (@error, @project, @privilegeLevel, @protocolVersion) => - done() + async.series [ + (cb) => + FixturesManager.setUpProject { + privilegeLevel: null + project: { + name: "Test Project" + } + }, (e, {@project_id, @user_id}) => + cb(e) + + (cb) => + @client = RealTimeClient.connect() + @client.on "connect", cb + + (cb) => + @client.emit "joinProject", project_id: @project_id, (@error, @project, @privilegeLevel, @protocolVersion) => + cb() + ], done it "should return an error", -> # We don't return specific errors diff --git a/services/real-time/test/acceptance/coffee/LeaveDocTests.coffee b/services/real-time/test/acceptance/coffee/LeaveDocTests.coffee index fe1ef94f4d..8c676647d5 100644 --- a/services/real-time/test/acceptance/coffee/LeaveDocTests.coffee +++ b/services/real-time/test/acceptance/coffee/LeaveDocTests.coffee @@ -6,6 +6,8 @@ RealTimeClient = require "./helpers/RealTimeClient" MockDocUpdaterServer = require "./helpers/MockDocUpdaterServer" FixturesManager = require "./helpers/FixturesManager" +async = require "async" + describe "leaveDoc", -> before -> @lines = ["test", "doc", "lines"] @@ -14,20 +16,27 @@ describe "leaveDoc", -> describe "when joined to a doc", -> before (done) -> - FixturesManager.setUpProject { - privilegeLevel: "readAndWrite" - }, (error, data) => - throw error if error? - {@project_id, @user_id} = data - FixturesManager.setUpDoc @project_id, {@lines, @version, @ops}, (error, data) => - throw error if error? - {@doc_id} = data + async.series [ + (cb) => + FixturesManager.setUpProject { + privilegeLevel: "readAndWrite" + }, (e, {@project_id, @user_id}) => + cb(e) + + (cb) => + FixturesManager.setUpDoc @project_id, {@lines, @version, @ops}, (e, {@doc_id}) => + cb(e) + + (cb) => @client = RealTimeClient.connect() - @client.emit "joinProject", project_id: @project_id, (error) => - throw error if error? - @client.emit "joinDoc", @doc_id, (error, @returnedArgs...) => - throw error if error? - done() + @client.on "connect", cb + + (cb) => + @client.emit "joinProject", project_id: @project_id, cb + + (cb) => + @client.emit "joinDoc", @doc_id, (error, @returnedArgs...) => cb(error) + ], done describe "then leaving the doc", -> before (done) -> From 347ceaaf03728b3c8e5584eb99697e1e75e1183f Mon Sep 17 00:00:00 2001 From: James Allen Date: Fri, 14 Nov 2014 15:30:18 +0000 Subject: [PATCH 017/491] Listen for updates from doc updater and send them to clients --- services/real-time/app.coffee | 3 + .../coffee/DocumentUpdaterController.coffee | 39 +++++++ .../coffee/ReceiveUpdateTests.coffee | 101 +++++++++++++++++ .../DocumentUpdaterControllerTests.coffee | 103 ++++++++++++++++++ .../coffee/WebsocketControllerTests.coffee | 6 +- .../unit/coffee/helpers/MockClient.coffee | 17 +++ 6 files changed, 267 insertions(+), 2 deletions(-) create mode 100644 services/real-time/app/coffee/DocumentUpdaterController.coffee create mode 100644 services/real-time/test/acceptance/coffee/ReceiveUpdateTests.coffee create mode 100644 services/real-time/test/unit/coffee/DocumentUpdaterControllerTests.coffee create mode 100644 services/real-time/test/unit/coffee/helpers/MockClient.coffee diff --git a/services/real-time/app.coffee b/services/real-time/app.coffee index 016aa73b5b..b6d8c2e23c 100644 --- a/services/real-time/app.coffee +++ b/services/real-time/app.coffee @@ -44,6 +44,9 @@ Router.configure(app, io, sessionSockets) WebsocketLoadBalancer = require "./app/js/WebsocketLoadBalancer" WebsocketLoadBalancer.listenForEditorEvents(io) + +DocumentUpdaterController = require "./app/js/DocumentUpdaterController" +DocumentUpdaterController.listenForUpdatesFromDocumentUpdater(io) port = Settings.internal.realTime.port host = Settings.internal.realTime.host diff --git a/services/real-time/app/coffee/DocumentUpdaterController.coffee b/services/real-time/app/coffee/DocumentUpdaterController.coffee new file mode 100644 index 0000000000..3de0becfc9 --- /dev/null +++ b/services/real-time/app/coffee/DocumentUpdaterController.coffee @@ -0,0 +1,39 @@ +logger = require "logger-sharelatex" +settings = require 'settings-sharelatex' +redis = require("redis-sharelatex") +rclient = redis.createClient(settings.redis.web) + +module.exports = DocumentUpdaterController = + # DocumentUpdaterController is responsible for updates that come via Redis + # Pub/Sub from the document updater. + + listenForUpdatesFromDocumentUpdater: (io) -> + rclient.subscribe "applied-ops" + rclient.on "message", (channel, message) -> + DocumentUpdaterController._processMessageFromDocumentUpdater(io, channel, message) + + _processMessageFromDocumentUpdater: (io, channel, message) -> + message = JSON.parse message + if message.op? + DocumentUpdaterController._applyUpdateFromDocumentUpdater(io, message.doc_id, message.op) + else if message.error? + DocumentUpdaterController._processErrorFromDocumentUpdater(io, message.doc_id, message.error, message) + + _applyUpdateFromDocumentUpdater: (io, doc_id, update) -> + for client in io.sockets.clients(doc_id) + if client.id == update.meta.source + logger.log doc_id: doc_id, version: update.v, source: update.meta?.source, "distributing update to sender" + client.emit "otUpdateApplied", v: update.v, doc: update.doc + else + logger.log doc_id: doc_id, version: update.v, source: update.meta?.source, client_id: client.id, "distributing update to collaborator" + client.emit "otUpdateApplied", update + + _processErrorFromDocumentUpdater: (io, doc_id, error, message) -> + logger.error err: error, doc_id: doc_id, "error from document updater" + for client in io.sockets.clients(doc_id) + client.emit "otUpdateError", error, message + client.disconnect() + + + + diff --git a/services/real-time/test/acceptance/coffee/ReceiveUpdateTests.coffee b/services/real-time/test/acceptance/coffee/ReceiveUpdateTests.coffee new file mode 100644 index 0000000000..9c95567d93 --- /dev/null +++ b/services/real-time/test/acceptance/coffee/ReceiveUpdateTests.coffee @@ -0,0 +1,101 @@ +chai = require("chai") +expect = chai.expect +chai.should() + +RealTimeClient = require "./helpers/RealTimeClient" +MockWebServer = require "./helpers/MockWebServer" +FixturesManager = require "./helpers/FixturesManager" + +async = require "async" + +settings = require "settings-sharelatex" +redis = require "redis-sharelatex" +rclient = redis.createClient(settings.redis.web) + +describe "receiveUpdate", -> + before (done) -> + @lines = ["test", "doc", "lines"] + @version = 42 + @ops = ["mock", "doc", "ops"] + + async.series [ + (cb) => + FixturesManager.setUpProject { + privilegeLevel: "owner" + project: { name: "Test Project" } + }, (error, {@user_id, @project_id}) => cb() + + (cb) => + FixturesManager.setUpDoc @project_id, {@lines, @version, @ops}, (e, {@doc_id}) => + cb(e) + + (cb) => + @clientA = RealTimeClient.connect() + @clientA.on "connect", cb + + (cb) => + @clientB = RealTimeClient.connect() + @clientB.on "connect", cb + + (cb) => + @clientA.emit "joinProject", { + project_id: @project_id + }, cb + + (cb) => + @clientA.emit "joinDoc", @doc_id, cb + + (cb) => + @clientB.emit "joinProject", { + project_id: @project_id + }, cb + + (cb) => + @clientB.emit "joinDoc", @doc_id, cb + ], done + + describe "with an update from clientA", -> + before (done) -> + @clientAUpdates = [] + @clientA.on "otUpdateApplied", (update) => @clientAUpdates.push(update) + @clientBUpdates = [] + @clientB.on "otUpdateApplied", (update) => @clientBUpdates.push(update) + + @update = { + doc_id: @doc_id + op: + meta: + source: @clientA.socket.sessionid + v: @version + doc: @doc_id + op: [{i: "foo", p: 50}] + } + rclient.publish "applied-ops", JSON.stringify(@update) + setTimeout done, 200 # Give clients time to get message + + it "should send the full op to clientB", -> + @clientBUpdates.should.deep.equal [@update.op] + + it "should send an ack to clientA", -> + @clientAUpdates.should.deep.equal [{ + v: @version, doc: @doc_id + }] + + describe "with an error", -> + + before (done) -> + @clientAErrors = [] + @clientA.on "otUpdateError", (error) => @clientAErrors.push(error) + @clientBErrors = [] + @clientB.on "otUpdateError", (error) => @clientBErrors.push(error) + + rclient.publish "applied-ops", JSON.stringify({doc_id: @doc_id, error: @error = "something went wrong"}) + setTimeout done, 200 # Give clients time to get message + + it "should send the error to both clients", -> + @clientAErrors.should.deep.equal [@error] + @clientBErrors.should.deep.equal [@error] + + it "should disconnect the clients", -> + @clientA.socket.connected.should.equal false + @clientB.socket.connected.should.equal false \ No newline at end of file diff --git a/services/real-time/test/unit/coffee/DocumentUpdaterControllerTests.coffee b/services/real-time/test/unit/coffee/DocumentUpdaterControllerTests.coffee new file mode 100644 index 0000000000..479231743c --- /dev/null +++ b/services/real-time/test/unit/coffee/DocumentUpdaterControllerTests.coffee @@ -0,0 +1,103 @@ +SandboxedModule = require('sandboxed-module') +sinon = require('sinon') +require('chai').should() +modulePath = require('path').join __dirname, '../../../app/js/DocumentUpdaterController' +MockClient = require "./helpers/MockClient" + +describe "DocumentUpdaterController", -> + beforeEach -> + @project_id = "project-id-123" + @doc_id = "doc-id-123" + @callback = sinon.stub() + @io = { "mock": "socket.io" } + @EditorUpdatesController = SandboxedModule.require modulePath, requires: + "logger-sharelatex": @logger = { error: sinon.stub(), log: sinon.stub() } + "settings-sharelatex": @settings = + redis: web: {} + "redis-sharelatex" : + createClient: ()=> + @rclient = {auth:->} + + describe "listenForUpdatesFromDocumentUpdater", -> + beforeEach -> + @rclient.subscribe = sinon.stub() + @rclient.on = sinon.stub() + @EditorUpdatesController.listenForUpdatesFromDocumentUpdater() + + it "should subscribe to the doc-updater stream", -> + @rclient.subscribe.calledWith("applied-ops").should.equal true + + it "should register a callback to handle updates", -> + @rclient.on.calledWith("message").should.equal true + + describe "_processMessageFromDocumentUpdater", -> + describe "with update", -> + beforeEach -> + @message = + doc_id: @doc_id + op: {t: "foo", p: 12} + @EditorUpdatesController._applyUpdateFromDocumentUpdater = sinon.stub() + @EditorUpdatesController._processMessageFromDocumentUpdater @io, "applied-ops", JSON.stringify(@message) + + it "should apply the update", -> + @EditorUpdatesController._applyUpdateFromDocumentUpdater + .calledWith(@io, @doc_id, @message.op) + .should.equal true + + describe "with error", -> + beforeEach -> + @message = + doc_id: @doc_id + error: "Something went wrong" + @EditorUpdatesController._processErrorFromDocumentUpdater = sinon.stub() + @EditorUpdatesController._processMessageFromDocumentUpdater @io, "applied-ops", JSON.stringify(@message) + + it "should process the error", -> + @EditorUpdatesController._processErrorFromDocumentUpdater + .calledWith(@io, @doc_id, @message.error) + .should.equal true + + describe "_applyUpdateFromDocumentUpdater", -> + beforeEach -> + @sourceClient = new MockClient() + @otherClients = [new MockClient(), new MockClient()] + @update = + op: [ t: "foo", p: 12 ] + meta: source: @sourceClient.id + v: @version = 42 + doc: @doc_id + @io.sockets = + clients: sinon.stub().returns([@sourceClient, @otherClients...]) + @EditorUpdatesController._applyUpdateFromDocumentUpdater @io, @doc_id, @update + + it "should send a version bump to the source client", -> + @sourceClient.emit + .calledWith("otUpdateApplied", v: @version, doc: @doc_id) + .should.equal true + + it "should get the clients connected to the document", -> + @io.sockets.clients + .calledWith(@doc_id) + .should.equal true + + it "should send the full update to the other clients", -> + for client in @otherClients + client.emit + .calledWith("otUpdateApplied", @update) + .should.equal true + + describe "_processErrorFromDocumentUpdater", -> + beforeEach -> + @clients = [new MockClient(), new MockClient()] + @io.sockets = + clients: sinon.stub().returns(@clients) + @EditorUpdatesController._processErrorFromDocumentUpdater @io, @doc_id, "Something went wrong" + + it "should log out an error", -> + @logger.error.called.should.equal true + + it "should disconnect all clients in that document", -> + @io.sockets.clients.calledWith(@doc_id).should.equal true + for client in @clients + client.disconnect.called.should.equal true + diff --git a/services/real-time/test/unit/coffee/WebsocketControllerTests.coffee b/services/real-time/test/unit/coffee/WebsocketControllerTests.coffee index 8ea62f466b..db7832cf05 100644 --- a/services/real-time/test/unit/coffee/WebsocketControllerTests.coffee +++ b/services/real-time/test/unit/coffee/WebsocketControllerTests.coffee @@ -345,8 +345,10 @@ describe 'WebsocketController', -> @AuthorizationManager.assertClientCanEditProject = sinon.stub().callsArgWith(1, @error = new Error("not authorized")) @WebsocketController.applyOtUpdate @client, @doc_id, @update, @callback - it "should disconnect the client", -> - @client.disconnect.called.should.equal true + # This happens in a setTimeout to allow the client a chance to receive the error first. + # I'm not sure how to unit test, but it is acceptance tested. + # it "should disconnect the client", -> + # @client.disconnect.called.should.equal true it "should log an error", -> @logger.error.called.should.equal true diff --git a/services/real-time/test/unit/coffee/helpers/MockClient.coffee b/services/real-time/test/unit/coffee/helpers/MockClient.coffee new file mode 100644 index 0000000000..82c3c02b19 --- /dev/null +++ b/services/real-time/test/unit/coffee/helpers/MockClient.coffee @@ -0,0 +1,17 @@ +sinon = require('sinon') + +idCounter = 0 + +module.exports = class MockClient + constructor: () -> + @attributes = {} + @join = sinon.stub() + @emit = sinon.stub() + @disconnect = sinon.stub() + @id = idCounter++ + set : (key, value, callback) -> + @attributes[key] = value + callback() if callback? + get : (key, callback) -> + callback null, @attributes[key] + disconnect: () -> From fd56655529e35528b068385f0b0bea6134aed850 Mon Sep 17 00:00:00 2001 From: James Allen Date: Fri, 14 Nov 2014 15:53:59 +0000 Subject: [PATCH 018/491] Add in track changes and doc updater flushing calls --- .../app/coffee/DocumentUpdaterManager.coffee | 18 +++++++ .../app/coffee/TrackChangesManager.coffee | 16 +++++++ .../coffee/ReceiveUpdateTests.coffee | 1 - .../coffee/DocumentUpdaterManagerTests.coffee | 36 ++++++++++++++ .../coffee/TrackChangesManagerTests.coffee | 48 +++++++++++++++++++ 5 files changed, 118 insertions(+), 1 deletion(-) create mode 100644 services/real-time/app/coffee/TrackChangesManager.coffee create mode 100644 services/real-time/test/unit/coffee/TrackChangesManagerTests.coffee diff --git a/services/real-time/app/coffee/DocumentUpdaterManager.coffee b/services/real-time/app/coffee/DocumentUpdaterManager.coffee index 2af68eef0c..8610b454ec 100644 --- a/services/real-time/app/coffee/DocumentUpdaterManager.coffee +++ b/services/real-time/app/coffee/DocumentUpdaterManager.coffee @@ -27,6 +27,24 @@ module.exports = DocumentUpdaterManager = err.statusCode = res.statusCode logger.error {err, project_id, doc_id, url}, "doc updater returned a non-success status code: #{res.statusCode}" callback err + + flushProjectToMongoAndDelete: (project_id, callback = ()->) -> + logger.log project_id:project_id, "deleting project from document updater" + #timer = new metrics.Timer("delete.mongo.project") + url = "#{settings.apis.documentupdater.url}/project/#{project_id}" + request.del url, (err, res, body)-> + #timer.done() + if err? + logger.error {err, project_id}, "error deleting project from document updater" + return callback(err) + else if 200 <= res.statusCode < 300 + logger.log {project_id}, "deleted project from document updater" + return callback(null) + else + err = new Error("document updater returned a failure status code: #{res.statusCode}") + err.statusCode = res.statusCode + logger.error {err, project_id}, "document updater returned failure status code: #{res.statusCode}" + return callback(err) queueChange: (project_id, doc_id, change, callback = ()->)-> jsonChange = JSON.stringify change diff --git a/services/real-time/app/coffee/TrackChangesManager.coffee b/services/real-time/app/coffee/TrackChangesManager.coffee new file mode 100644 index 0000000000..49fa956cf6 --- /dev/null +++ b/services/real-time/app/coffee/TrackChangesManager.coffee @@ -0,0 +1,16 @@ +settings = require "settings-sharelatex" +request = require "request" +logger = require "logger-sharelatex" + +module.exports = TrackChangesManager = + flushProject: (project_id, callback = (error) ->) -> + logger.log project_id: project_id, "flushing project in track-changes api" + url = "#{settings.apis.trackchanges.url}/project/#{project_id}/flush" + request.post url, (error, res, body) -> + return callback(error) if error? + if 200 <= res.statusCode < 300 + callback(null) + else + error = new Error("track-changes api responded with non-success code: #{res.statusCode}") + logger.error err: error, project_id: project_id, "error flushing project in track-changes api" + callback(error) \ No newline at end of file diff --git a/services/real-time/test/acceptance/coffee/ReceiveUpdateTests.coffee b/services/real-time/test/acceptance/coffee/ReceiveUpdateTests.coffee index 9c95567d93..fd1b9a0dfe 100644 --- a/services/real-time/test/acceptance/coffee/ReceiveUpdateTests.coffee +++ b/services/real-time/test/acceptance/coffee/ReceiveUpdateTests.coffee @@ -82,7 +82,6 @@ describe "receiveUpdate", -> }] describe "with an error", -> - before (done) -> @clientAErrors = [] @clientA.on "otUpdateError", (error) => @clientAErrors.push(error) diff --git a/services/real-time/test/unit/coffee/DocumentUpdaterManagerTests.coffee b/services/real-time/test/unit/coffee/DocumentUpdaterManagerTests.coffee index 80caa45802..8a9d335b1a 100644 --- a/services/real-time/test/unit/coffee/DocumentUpdaterManagerTests.coffee +++ b/services/real-time/test/unit/coffee/DocumentUpdaterManagerTests.coffee @@ -62,6 +62,42 @@ describe 'DocumentUpdaterManager', -> .calledWith(err) .should.equal true + describe 'flushProjectToMongoAndDelete', -> + beforeEach -> + @callback = sinon.stub() + + describe "successfully", -> + beforeEach -> + @request.del = sinon.stub().callsArgWith(1, null, {statusCode: 204}, "") + @DocumentUpdaterManager.flushProjectToMongoAndDelete @project_id, @callback + + it 'should delete the project from the document updater', -> + url = "#{@settings.apis.documentupdater.url}/project/#{@project_id}" + @request.del.calledWith(url).should.equal true + + it "should call the callback with no error", -> + @callback.calledWith(null).should.equal true + + describe "when the document updater API returns an error", -> + beforeEach -> + @request.del = sinon.stub().callsArgWith(1, @error = new Error("something went wrong"), null, null) + @DocumentUpdaterManager.flushProjectToMongoAndDelete @project_id, @callback + + it "should return an error to the callback", -> + @callback.calledWith(@error).should.equal true + + describe "when the document updater returns a failure error code", -> + beforeEach -> + @request.del = sinon.stub().callsArgWith(1, null, { statusCode: 500 }, "") + @DocumentUpdaterManager.flushProjectToMongoAndDelete @project_id, @callback + + it "should return the callback with an error", -> + err = new Error("doc updater returned failure status code: 500") + err.statusCode = 500 + @callback + .calledWith(err) + .should.equal true + describe 'queueChange', -> beforeEach -> @change = { diff --git a/services/real-time/test/unit/coffee/TrackChangesManagerTests.coffee b/services/real-time/test/unit/coffee/TrackChangesManagerTests.coffee new file mode 100644 index 0000000000..69c11c87b1 --- /dev/null +++ b/services/real-time/test/unit/coffee/TrackChangesManagerTests.coffee @@ -0,0 +1,48 @@ +chai = require('chai') +chai.should() +sinon = require("sinon") +modulePath = "../../../app/js/TrackChangesManager" +SandboxedModule = require('sandboxed-module') + +describe "TrackChangesManager", -> + beforeEach -> + @TrackChangesManager = SandboxedModule.require modulePath, requires: + "request" : @request = sinon.stub() + "settings-sharelatex": @settings = + apis: + trackchanges: + url: "trackchanges.sharelatex.com" + "logger-sharelatex": @logger = {log: sinon.stub(), error: sinon.stub()} + @project_id = "project-id-123" + @callback = sinon.stub() + + describe "flushProject", -> + describe "with a successful response code", -> + beforeEach -> + @request.post = sinon.stub().callsArgWith(1, null, statusCode: 204, "") + @TrackChangesManager.flushProject @project_id, @callback + + it "should flush the project in the track changes api", -> + @request.post + .calledWith("#{@settings.apis.trackchanges.url}/project/#{@project_id}/flush") + .should.equal true + + it "should call the callback without an error", -> + @callback.calledWith(null).should.equal true + + describe "with a failed response code", -> + beforeEach -> + @request.post = sinon.stub().callsArgWith(1, null, statusCode: 500, "") + @TrackChangesManager.flushProject @project_id, @callback + + it "should call the callback with an error", -> + @callback.calledWith(new Error("track-changes api responded with a non-success code: 500")).should.equal true + + it "should log the error", -> + @logger.error + .calledWith({ + err: new Error("track-changes api responded with a non-success code: 500") + project_id: @project_id + }, "error flushing project in track-changes api") + .should.equal true + From 80b7875414d6d15f1be01b3a98781d5d2b3e28cb Mon Sep 17 00:00:00 2001 From: James Allen Date: Fri, 14 Nov 2014 16:51:55 +0000 Subject: [PATCH 019/491] Add in leaveProject handler --- services/real-time/Gruntfile.coffee | 1 + .../app/coffee/WebsocketController.coffee | 29 ++++++++++ .../coffee/ClientTrackingTests.coffee | 2 +- .../coffee/LeaveProjectTests.coffee | 16 ++++++ .../coffee/WebsocketControllerTests.coffee | 56 +++++++++++++++++++ 5 files changed, 103 insertions(+), 1 deletion(-) create mode 100644 services/real-time/test/acceptance/coffee/LeaveProjectTests.coffee diff --git a/services/real-time/Gruntfile.coffee b/services/real-time/Gruntfile.coffee index e01c8aecb1..28cc4233b0 100644 --- a/services/real-time/Gruntfile.coffee +++ b/services/real-time/Gruntfile.coffee @@ -44,6 +44,7 @@ module.exports = (grunt) -> unit: options: reporter: grunt.option('reporter') or 'spec' + grep: grunt.option("grep") src: ["test/unit/js/**/*.js"] acceptance: options: diff --git a/services/real-time/app/coffee/WebsocketController.coffee b/services/real-time/app/coffee/WebsocketController.coffee index 6f1b3b5119..04a9d926de 100644 --- a/services/real-time/app/coffee/WebsocketController.coffee +++ b/services/real-time/app/coffee/WebsocketController.coffee @@ -3,6 +3,7 @@ WebApiManager = require "./WebApiManager" AuthorizationManager = require "./AuthorizationManager" DocumentUpdaterManager = require "./DocumentUpdaterManager" ConnectedUsersManager = require "./ConnectedUsersManager" +TrackChangesManager = require "./TrackChangesManager" WebsocketLoadBalancer = require "./WebsocketLoadBalancer" Utils = require "./Utils" @@ -40,6 +41,34 @@ module.exports = WebsocketController = # No need to block for setting the user as connected in the cursor tracking ConnectedUsersManager.updateUserPosition project_id, client.id, user, null, () -> + + # We want to flush a project if there are no more (local) connected clients + # but we need to wait for the triggering client to disconnect. How long we wait + # is determined by FLUSH_IF_EMPTY_DELAY. + FLUSH_IF_EMPTY_DELAY: 500 #ms + leaveProject: (io, client, callback = (error) ->) -> + Utils.getClientAttributes client, ["project_id", "user_id"], (error, {project_id, user_id}) -> + return callback(error) if error? + logger.log {project_id, user_id, client_id: client.id}, "client leaving project" + WebsocketLoadBalancer.emitToRoom project_id, "clientTracking.clientDisconnected", client.id + + # We can do this in the background + ConnectedUsersManager.markUserAsDisconnected project_id, client.id, (err) -> + if err? + logger.error {err, project_id, user_id, client_id: client.id}, "error marking client as disconnected" + + setTimeout () -> + remainingClients = io.sockets.clients(project_id) + if remainingClients.length == 0 + # Flush project in the background + DocumentUpdaterManager.flushProjectToMongoAndDelete project_id, (err) -> + if err? + logger.error {err, project_id, user_id, client_id: client.id}, "error flushing to doc updater after leaving project" + TrackChangesManager.flushProject project_id, (err) -> + if err? + logger.error {err, project_id, user_id, client_id: client.id}, "error flushing to track changes after leaving project" + callback() + , WebsocketController.FLUSH_IF_EMPTY_DELAY joinDoc: (client, doc_id, fromVersion = -1, callback = (error, doclines, version, ops) ->) -> Utils.getClientAttributes client, ["project_id", "user_id"], (error, {project_id, user_id}) -> diff --git a/services/real-time/test/acceptance/coffee/ClientTrackingTests.coffee b/services/real-time/test/acceptance/coffee/ClientTrackingTests.coffee index c0f0079bbc..90071e61ec 100644 --- a/services/real-time/test/acceptance/coffee/ClientTrackingTests.coffee +++ b/services/real-time/test/acceptance/coffee/ClientTrackingTests.coffee @@ -48,7 +48,7 @@ describe "clientTracking", -> doc_id: @doc_id = "mock-doc-id" }, (error) -> throw error if error? - done() + setTimeout done, 300 # Give the message a chance to reach client B. it "should tell other clients about the update", -> @updates.should.deep.equal [ diff --git a/services/real-time/test/acceptance/coffee/LeaveProjectTests.coffee b/services/real-time/test/acceptance/coffee/LeaveProjectTests.coffee new file mode 100644 index 0000000000..9698abd2c2 --- /dev/null +++ b/services/real-time/test/acceptance/coffee/LeaveProjectTests.coffee @@ -0,0 +1,16 @@ +describe "leaveProject", -> + describe "with other clients in the project", -> + it "should emit a disconnect message to the room" + + it "should no longer list the client in connected users" + + it "should not flush the project to the document updater" + + it "should not flush the project in track changes" + + describe "with no other clients in the project", -> + it "should flush the project to the document updater" + + it "should flush the project in track changes" + + \ No newline at end of file diff --git a/services/real-time/test/unit/coffee/WebsocketControllerTests.coffee b/services/real-time/test/unit/coffee/WebsocketControllerTests.coffee index db7832cf05..3ff8b28f3b 100644 --- a/services/real-time/test/unit/coffee/WebsocketControllerTests.coffee +++ b/services/real-time/test/unit/coffee/WebsocketControllerTests.coffee @@ -29,6 +29,7 @@ describe 'WebsocketController', -> "./WebApiManager": @WebApiManager = {} "./AuthorizationManager": @AuthorizationManager = {} "./DocumentUpdaterManager": @DocumentUpdaterManager = {} + "./TrackChangesManager": @TrackChangesManager = {} "./ConnectedUsersManager": @ConnectedUsersManager = {} "./WebsocketLoadBalancer": @WebsocketLoadBalancer = {} "logger-sharelatex": @logger = { log: sinon.stub(), error: sinon.stub() } @@ -108,7 +109,62 @@ describe 'WebsocketController', -> @callback .calledWith(new Error("not authorized")) .should.equal true + + describe "leaveProject", -> + beforeEach -> + @DocumentUpdaterManager.flushProjectToMongoAndDelete = sinon.stub().callsArg(1) + @TrackChangesManager.flushProject = sinon.stub().callsArg(1) + @ConnectedUsersManager.markUserAsDisconnected = sinon.stub().callsArg(2) + @WebsocketLoadBalancer.emitToRoom = sinon.stub() + @clientsInRoom = [] + @io = + sockets: + clients: (room_id) => + if room_id != @project_id + throw "expected room_id to be project_id" + return @clientsInRoom + @client.params.project_id = @project_id + @WebsocketController.FLUSH_IF_EMPTY_DELAY = 0 + tk.reset() # Allow setTimeout to work. + + describe "when the project is empty", -> + beforeEach (done) -> + @clientsInRoom = [] + @WebsocketController.leaveProject @io, @client, done + + it "should end clientTracking.clientDisconnected to the project room", -> + @WebsocketLoadBalancer.emitToRoom + .calledWith(@project_id, "clientTracking.clientDisconnected", @client.id) + .should.equal true + + it "should mark the user as disconnected", -> + @ConnectedUsersManager.markUserAsDisconnected + .calledWith(@project_id, @client.id) + .should.equal true + + it "should flush the project in the document updater", -> + @DocumentUpdaterManager.flushProjectToMongoAndDelete + .calledWith(@project_id) + .should.equal true + it "should flush the changes in the track changes api", -> + @TrackChangesManager.flushProject + .calledWith(@project_id) + .should.equal true + + describe "when the project is not empty", -> + beforeEach -> + @clientsInRoom = ["mock-remaining-client"] + @WebsocketController.leaveProject @io, @client + + it "should not flush the project in the document updater", -> + @DocumentUpdaterManager.flushProjectToMongoAndDelete + .called.should.equal false + + it "should not flush the changes in the track changes api", -> + @TrackChangesManager.flushProject + .called.should.equal false + describe "joinDoc", -> beforeEach -> @doc_id = "doc-id-123" From 7b275e9e0efa5cee13def8002556d39f0fd48035 Mon Sep 17 00:00:00 2001 From: James Allen Date: Mon, 17 Nov 2014 12:23:30 +0000 Subject: [PATCH 020/491] Add acceptence tests for leaving(disconnecting) from a project --- services/real-time/app/coffee/Router.coffee | 8 +- .../real-time/config/settings.defaults.coffee | 2 + .../coffee/LeaveProjectTests.coffee | 112 ++++++++++++++++-- .../helpers/MockDocUpdaterServer.coffee | 9 ++ .../helpers/MockTrackChangesServer.coffee | 21 ++++ 5 files changed, 143 insertions(+), 9 deletions(-) create mode 100644 services/real-time/test/acceptance/coffee/helpers/MockTrackChangesServer.coffee diff --git a/services/real-time/app/coffee/Router.coffee b/services/real-time/app/coffee/Router.coffee index 7872727377..85a7940b7a 100644 --- a/services/real-time/app/coffee/Router.coffee +++ b/services/real-time/app/coffee/Router.coffee @@ -5,7 +5,7 @@ HttpController = require "./HttpController" Utils = require "./Utils" module.exports = Router = - _handleError: (callback, error, client, method, extraAttrs = {}) -> + _handleError: (callback = ((error) ->), error, client, method, extraAttrs = {}) -> Utils.getClientAttributes client, ["project_id", "doc_id", "user_id"], (_, attrs) -> for key, value of extraAttrs attrs[key] = value @@ -13,7 +13,6 @@ module.exports = Router = attrs.err = error logger.error attrs, "server side error in #{method}" # Don't return raw error to prevent leaking server side info - console.log "CALLING CALLBACK", callback return callback {message: "Something went wrong"} configure: (app, io, session) -> @@ -44,6 +43,11 @@ module.exports = Router = else callback(null, args...) + client.on "disconnect", () -> + WebsocketController.leaveProject io, client, (err) -> + if err? + Router._handleError null, err, client, "leaveProject" + client.on "joinDoc", (doc_id, fromVersion, callback) -> # fromVersion is optional diff --git a/services/real-time/config/settings.defaults.coffee b/services/real-time/config/settings.defaults.coffee index 7d9ddf184c..ccaa02e523 100644 --- a/services/real-time/config/settings.defaults.coffee +++ b/services/real-time/config/settings.defaults.coffee @@ -15,6 +15,8 @@ module.exports = url: "http://localhost:3000" documentupdater: url: "http://localhost:3003" + trackchanges: + url: "http://localhost:3015" security: sessionSecret: "secret-please-change" diff --git a/services/real-time/test/acceptance/coffee/LeaveProjectTests.coffee b/services/real-time/test/acceptance/coffee/LeaveProjectTests.coffee index 9698abd2c2..2d1361a18b 100644 --- a/services/real-time/test/acceptance/coffee/LeaveProjectTests.coffee +++ b/services/real-time/test/acceptance/coffee/LeaveProjectTests.coffee @@ -1,16 +1,114 @@ +RealTimeClient = require "./helpers/RealTimeClient" +MockDocUpdaterServer = require "./helpers/MockDocUpdaterServer" +MockTrackChangesServer = require "./helpers/MockTrackChangesServer" +FixturesManager = require "./helpers/FixturesManager" + +async = require "async" + describe "leaveProject", -> + before (done) -> + MockDocUpdaterServer.run (error) -> + return done(error) if error? + MockTrackChangesServer.run done + describe "with other clients in the project", -> - it "should emit a disconnect message to the room" + before (done) -> + async.series [ + (cb) => + FixturesManager.setUpProject { + privilegeLevel: "owner" + project: { + name: "Test Project" + } + }, (e, {@project_id, @user_id}) => cb() + + (cb) => + @clientA = RealTimeClient.connect() + @clientA.on "connect", cb + + (cb) => + @clientB = RealTimeClient.connect() + @clientB.on "connect", cb + + @clientBDisconnectMessages = [] + @clientB.on "clientTracking.clientDisconnected", (data) => + @clientBDisconnectMessages.push data + + (cb) => + @clientA.emit "joinProject", project_id: @project_id, (error, @project, @privilegeLevel, @protocolVersion) => + cb(error) + + (cb) => + @clientB.emit "joinProject", project_id: @project_id, (error, @project, @privilegeLevel, @protocolVersion) => + cb(error) + + (cb) => + # leaveProject is called when the client disconnects + @clientA.on "disconnect", () -> cb() + @clientA.disconnect() + + (cb) => + # The API waits a little while before flushing changes + setTimeout done, require("../../../app/js/WebsocketController").FLUSH_IF_EMPTY_DELAY * 2 + + ], done + + it "should emit a disconnect message to the room", -> + @clientBDisconnectMessages.should.deep.equal [@clientA.socket.sessionid] - it "should no longer list the client in connected users" + it "should no longer list the client in connected users", (done) -> + @clientB.emit "clientTracking.getConnectedUsers", (error, users) => + for user in users + if user.client_id == @clientA.socket.sessionid + throw "Expected clientA to not be listed in connected users" + return done() - it "should not flush the project to the document updater" + it "should not flush the project to the document updater", -> + MockDocUpdaterServer.deleteProject + .calledWith(@project_id) + .should.equal false - it "should not flush the project in track changes" + it "should not flush the project in track changes", -> + MockTrackChangesServer.flushProject + .calledWith(@project_id) + .should.equal false describe "with no other clients in the project", -> - it "should flush the project to the document updater" - - it "should flush the project in track changes" + before (done) -> + async.series [ + (cb) => + FixturesManager.setUpProject { + privilegeLevel: "owner" + project: { + name: "Test Project" + } + }, (e, {@project_id, @user_id}) => cb() + + (cb) => + @clientA = RealTimeClient.connect() + @clientA.on "connect", cb + + (cb) => + @clientA.emit "joinProject", project_id: @project_id, (error, @project, @privilegeLevel, @protocolVersion) => + cb(error) + + (cb) => + # leaveProject is called when the client disconnects + @clientA.on "disconnect", () -> cb() + @clientA.disconnect() + + (cb) => + # The API waits a little while before flushing changes + setTimeout done, require("../../../app/js/WebsocketController").FLUSH_IF_EMPTY_DELAY * 2 + ], done + + it "should flush the project to the document updater", -> + MockDocUpdaterServer.deleteProject + .calledWith(@project_id) + .should.equal true + it "should flush the project in track changes", -> + MockTrackChangesServer.flushProject + .calledWith(@project_id) + .should.equal true \ No newline at end of file diff --git a/services/real-time/test/acceptance/coffee/helpers/MockDocUpdaterServer.coffee b/services/real-time/test/acceptance/coffee/helpers/MockDocUpdaterServer.coffee index 094d10aceb..0f196371b3 100644 --- a/services/real-time/test/acceptance/coffee/helpers/MockDocUpdaterServer.coffee +++ b/services/real-time/test/acceptance/coffee/helpers/MockDocUpdaterServer.coffee @@ -12,6 +12,8 @@ module.exports = MockDocUpdaterServer = null, MockDocUpdaterServer.docs["#{project_id}:#{doc_id}"] ) + deleteProject: sinon.stub().callsArg(1) + getDocumentRequest: (req, res, next) -> {project_id, doc_id} = req.params {fromVersion} = req.query @@ -19,6 +21,12 @@ module.exports = MockDocUpdaterServer = MockDocUpdaterServer.getDocument project_id, doc_id, fromVersion, (error, data) -> return next(error) if error? res.json data + + deleteProjectRequest: (req, res, next) -> + {project_id} = req.params + MockDocUpdaterServer.deleteProject project_id, (error) -> + return next(error) if error? + res.sendStatus 204 running: false run: (callback = (error) ->) -> @@ -26,6 +34,7 @@ module.exports = MockDocUpdaterServer = return callback() app = express() app.get "/project/:project_id/doc/:doc_id", MockDocUpdaterServer.getDocumentRequest + app.delete "/project/:project_id", MockDocUpdaterServer.deleteProjectRequest app.listen 3003, (error) -> MockDocUpdaterServer.running = true callback(error) diff --git a/services/real-time/test/acceptance/coffee/helpers/MockTrackChangesServer.coffee b/services/real-time/test/acceptance/coffee/helpers/MockTrackChangesServer.coffee new file mode 100644 index 0000000000..e7ddbf5cea --- /dev/null +++ b/services/real-time/test/acceptance/coffee/helpers/MockTrackChangesServer.coffee @@ -0,0 +1,21 @@ +sinon = require "sinon" +express = require "express" + +module.exports = MockTrackChangesServer = + flushProject: sinon.stub().callsArg(1) + + flushProjectRequest: (req, res, next) -> + {project_id} = req.params + MockTrackChangesServer.flushProject project_id, (error) -> + return next(error) if error? + res.sendStatus 204 + + running: false + run: (callback = (error) ->) -> + if MockTrackChangesServer.running + return callback() + app = express() + app.post "/project/:project_id/flush", MockTrackChangesServer.flushProjectRequest + app.listen 3015, (error) -> + MockTrackChangesServer.running = true + callback(error) \ No newline at end of file From 8bc6d0e291d3a9333bc5650ba5c8e1d552315099 Mon Sep 17 00:00:00 2001 From: James Allen Date: Mon, 17 Nov 2014 12:46:27 +0000 Subject: [PATCH 021/491] Unify logging --- .../app/coffee/WebsocketController.coffee | 44 ++++++++++--------- 1 file changed, 24 insertions(+), 20 deletions(-) diff --git a/services/real-time/app/coffee/WebsocketController.coffee b/services/real-time/app/coffee/WebsocketController.coffee index 04a9d926de..9315c0f15e 100644 --- a/services/real-time/app/coffee/WebsocketController.coffee +++ b/services/real-time/app/coffee/WebsocketController.coffee @@ -38,6 +38,7 @@ module.exports = WebsocketController = client.set("login_count", user?.loginCount) callback null, project, privilegeLevel, WebsocketController.PROTOCOL_VERSION + logger.log {user_id, project_id, client_id: client.id}, "user joined project" # No need to block for setting the user as connected in the cursor tracking ConnectedUsersManager.updateUserPosition project_id, client.id, user, null, () -> @@ -72,13 +73,12 @@ module.exports = WebsocketController = joinDoc: (client, doc_id, fromVersion = -1, callback = (error, doclines, version, ops) ->) -> Utils.getClientAttributes client, ["project_id", "user_id"], (error, {project_id, user_id}) -> + return callback(error) if error? + return callback(new Error("no project_id found on client")) if !project_id? logger.log {user_id, project_id, doc_id, fromVersion, client_id: client.id}, "client joining doc" - AuthorizationManager.assertClientCanViewProject client, (error) -> - return callback(error) if error? - client.get "project_id", (error, project_id) -> + AuthorizationManager.assertClientCanViewProject client, (error) -> return callback(error) if error? - return callback(new Error("no project_id found on client")) if !project_id? DocumentUpdaterManager.getDocument project_id, doc_id, fromVersion, (error, lines, version, ops) -> return callback(error) if error? # Encode any binary bits of data so it can go via WebSockets @@ -93,6 +93,7 @@ module.exports = WebsocketController = escapedLines.push line client.join(doc_id) callback null, escapedLines, version, ops + logger.log {user_id, project_id, doc_id, fromVersion, client_id: client.id}, "client joined doc" leaveDoc: (client, doc_id, callback = (error) ->) -> Utils.getClientAttributes client, ["project_id", "user_id"], (error, {project_id, user_id}) -> @@ -128,29 +129,31 @@ module.exports = WebsocketController = getConnectedUsers: (client, callback = (error, users) ->) -> Utils.getClientAttributes client, ["project_id", "user_id"], (error, {project_id, user_id}) -> - logger.log {user_id, project_id, client_id: client.id}, "getting connected users" - - AuthorizationManager.assertClientCanViewProject client, (error) -> return callback(error) if error? - client.get "project_id", (error, project_id) -> + return callback(new Error("no project_id found on client")) if !project_id? + logger.log {user_id, project_id, client_id: client.id}, "getting connected users" + AuthorizationManager.assertClientCanViewProject client, (error) -> return callback(error) if error? - return callback(new Error("no project_id found on client")) if !project_id? ConnectedUsersManager.getConnectedUsers project_id, (error, users) -> return callback(error) if error? callback null, users + logger.log {user_id, project_id, client_id: client.id}, "got connected users" + applyOtUpdate: (client, doc_id, update, callback = (error) ->) -> - AuthorizationManager.assertClientCanEditProject client, (error) -> - if error? - logger.error {err: error, doc_id, client_id: client.id, version: update.v}, "client is not authorized to make update" - setTimeout () -> - # Disconnect, but give the client the chance to receive the error - client.disconnect() - , 100 - return callback(error) - - Utils.getClientAttributes client, ["user_id", "project_id"], (error, {user_id, project_id}) -> - return callback(error) if error? + Utils.getClientAttributes client, ["user_id", "project_id"], (error, {user_id, project_id}) -> + return callback(error) if error? + return callback(new Error("no project_id found on client")) if !project_id? + # Omit this logging for now since it's likely too noisey + #logger.log {user_id, project_id, doc_id, client_id: client.id, update: update}, "applying update" + AuthorizationManager.assertClientCanEditProject client, (error) -> + if error? + logger.error {err: error, doc_id, client_id: client.id, version: update.v}, "client is not authorized to make update" + setTimeout () -> + # Disconnect, but give the client the chance to receive the error + client.disconnect() + , 100 + return callback(error) update.meta ||= {} update.meta.source = client.id update.meta.user_id = user_id @@ -165,3 +168,4 @@ module.exports = WebsocketController = logger.error {err: error, project_id, doc_id, client_id: client.id, version: update.v}, "document was not available for update" client.disconnect() callback(error) + #logger.log {user_id, project_id, doc_id, client_id: client.id}, "applied update" From 66dfafdebe0d777b96d2865973f457a16fccc855 Mon Sep 17 00:00:00 2001 From: James Allen Date: Mon, 17 Nov 2014 13:12:49 +0000 Subject: [PATCH 022/491] Add metrics into all end points --- services/real-time/app/coffee/Router.coffee | 5 ++- .../app/coffee/WebsocketController.coffee | 13 +++++-- .../coffee/WebsocketControllerTests.coffee | 38 +++++++++++++++---- 3 files changed, 43 insertions(+), 13 deletions(-) diff --git a/services/real-time/app/coffee/Router.coffee b/services/real-time/app/coffee/Router.coffee index 85a7940b7a..14d2f698a8 100644 --- a/services/real-time/app/coffee/Router.coffee +++ b/services/real-time/app/coffee/Router.coffee @@ -1,4 +1,4 @@ -Metrics = require "metrics-sharelatex" +metrics = require "metrics-sharelatex" logger = require "logger-sharelatex" WebsocketController = require "./WebsocketController" HttpController = require "./HttpController" @@ -26,7 +26,7 @@ module.exports = Router = client?.disconnect() return - Metrics.inc('socket-io.connection') + metrics.inc('socket-io.connection') logger.log session: session, client_id: client.id, "client connected" @@ -44,6 +44,7 @@ module.exports = Router = callback(null, args...) client.on "disconnect", () -> + metrics.inc('socket-io.disconnect') WebsocketController.leaveProject io, client, (err) -> if err? Router._handleError null, err, client, "leaveProject" diff --git a/services/real-time/app/coffee/WebsocketController.coffee b/services/real-time/app/coffee/WebsocketController.coffee index 9315c0f15e..73f327d22f 100644 --- a/services/real-time/app/coffee/WebsocketController.coffee +++ b/services/real-time/app/coffee/WebsocketController.coffee @@ -1,4 +1,5 @@ logger = require "logger-sharelatex" +metrics = require "metrics-sharelatex" WebApiManager = require "./WebApiManager" AuthorizationManager = require "./AuthorizationManager" DocumentUpdaterManager = require "./DocumentUpdaterManager" @@ -16,6 +17,7 @@ module.exports = WebsocketController = joinProject: (client, user, project_id, callback = (error, project, privilegeLevel, protocolVersion) ->) -> user_id = user?._id logger.log {user_id, project_id, client_id: client.id}, "user joining project" + metrics.inc "editor.join-project" WebApiManager.joinProject project_id, user_id, (error, project, privilegeLevel) -> return callback(error) if error? @@ -48,6 +50,7 @@ module.exports = WebsocketController = # is determined by FLUSH_IF_EMPTY_DELAY. FLUSH_IF_EMPTY_DELAY: 500 #ms leaveProject: (io, client, callback = (error) ->) -> + metrics.inc "editor.leave-project" Utils.getClientAttributes client, ["project_id", "user_id"], (error, {project_id, user_id}) -> return callback(error) if error? logger.log {project_id, user_id, client_id: client.id}, "client leaving project" @@ -72,6 +75,7 @@ module.exports = WebsocketController = , WebsocketController.FLUSH_IF_EMPTY_DELAY joinDoc: (client, doc_id, fromVersion = -1, callback = (error, doclines, version, ops) ->) -> + metrics.inc "editor.join-doc" Utils.getClientAttributes client, ["project_id", "user_id"], (error, {project_id, user_id}) -> return callback(error) if error? return callback(new Error("no project_id found on client")) if !project_id? @@ -96,12 +100,14 @@ module.exports = WebsocketController = logger.log {user_id, project_id, doc_id, fromVersion, client_id: client.id}, "client joined doc" leaveDoc: (client, doc_id, callback = (error) ->) -> + metrics.inc "editor.leave-doc" Utils.getClientAttributes client, ["project_id", "user_id"], (error, {project_id, user_id}) -> logger.log {user_id, project_id, doc_id, client_id: client.id}, "client leaving doc" client.leave doc_id callback() updateClientPosition: (client, cursorData, callback = (error) ->) -> + metrics.inc "editor.update-client-position", 0.1 Utils.getClientAttributes client, [ "project_id", "first_name", "last_name", "email", "user_id" ], (error, {project_id, first_name, last_name, email, user_id}) -> @@ -128,6 +134,7 @@ module.exports = WebsocketController = WebsocketLoadBalancer.emitToRoom(project_id, "clientTracking.clientUpdated", cursorData) getConnectedUsers: (client, callback = (error, users) ->) -> + metrics.inc "editor.get-connected-users" Utils.getClientAttributes client, ["project_id", "user_id"], (error, {project_id, user_id}) -> return callback(error) if error? return callback(new Error("no project_id found on client")) if !project_id? @@ -157,9 +164,9 @@ module.exports = WebsocketController = update.meta ||= {} update.meta.source = client.id update.meta.user_id = user_id - # metrics.inc "editor.doc-update", 0.3 - # metrics.set "editor.active-projects", project_id, 0.3 - # metrics.set "editor.active-users", user_id, 0.3 + metrics.inc "editor.doc-update", 0.3 + metrics.set "editor.active-projects", project_id, 0.3 + metrics.set "editor.active-users", user_id, 0.3 logger.log {user_id, doc_id, project_id, client_id: client.id, version: update.v}, "sending update to doc updater" diff --git a/services/real-time/test/unit/coffee/WebsocketControllerTests.coffee b/services/real-time/test/unit/coffee/WebsocketControllerTests.coffee index 3ff8b28f3b..5d2e63cc03 100644 --- a/services/real-time/test/unit/coffee/WebsocketControllerTests.coffee +++ b/services/real-time/test/unit/coffee/WebsocketControllerTests.coffee @@ -33,6 +33,10 @@ describe 'WebsocketController', -> "./ConnectedUsersManager": @ConnectedUsersManager = {} "./WebsocketLoadBalancer": @WebsocketLoadBalancer = {} "logger-sharelatex": @logger = { log: sinon.stub(), error: sinon.stub() } + "metrics-sharelatex": @metrics = + inc: sinon.stub() + set: sinon.stub() + afterEach -> tk.reset() @@ -99,6 +103,9 @@ describe 'WebsocketController', -> @ConnectedUsersManager.updateUserPosition .calledWith(@project_id, @client.id, @user, null) .should.equal true + + it "should increment the join-project metric", -> + @metrics.inc.calledWith("editor.join-project").should.equal true describe "when not authorized", -> beforeEach -> @@ -151,6 +158,9 @@ describe 'WebsocketController', -> @TrackChangesManager.flushProject .calledWith(@project_id) .should.equal true + + it "should increment the leave-project metric", -> + @metrics.inc.calledWith("editor.leave-project").should.equal true describe "when the project is not empty", -> beforeEach -> @@ -202,6 +212,9 @@ describe 'WebsocketController', -> .calledWith(null, @doc_lines, @version, @ops) .should.equal true + it "should increment the join-doc metric", -> + @metrics.inc.calledWith("editor.join-doc").should.equal true + describe "with doclines that need escaping", -> beforeEach -> @doc_lines.push ["räksmörgås"] @@ -238,6 +251,9 @@ describe 'WebsocketController', -> it "should call the callback", -> @callback.called.should.equal true + it "should increment the leave-doc metric", -> + @metrics.inc.calledWith("editor.leave-doc").should.equal true + describe "getConnectedUsers", -> beforeEach -> @client.params.project_id = @project_id @@ -261,6 +277,9 @@ describe 'WebsocketController', -> it "should return the users", -> @callback.calledWith(null, @users).should.equal true + + it "should increment the get-connected-users metric", -> + @metrics.inc.calledWith("editor.get-connected-users").should.equal true describe "when not authorized", -> beforeEach -> @@ -321,6 +340,9 @@ describe 'WebsocketController', -> doc_id: @doc_id }).should.equal true done() + + it "should increment the update-client-position metric at 0.1 frequency", -> + @metrics.inc.calledWith("editor.update-client-position", 0.1).should.equal true describe "with an anonymous user", -> beforeEach -> @@ -371,14 +393,14 @@ describe 'WebsocketController', -> it "should call the callback", -> @callback.called.should.equal true - # it "should update the active users metric", -> - # @metrics.set.calledWith("editor.active-users", @user_id).should.equal true - # - # it "should update the active projects metric", -> - # @metrics.set.calledWith("editor.active-projects", @project_id).should.equal true - # - # it "should increment the doc updates", -> - # @metrics.inc.calledWith("editor.doc-update").should.equal true + it "should update the active users metric", -> + @metrics.set.calledWith("editor.active-users", @user_id).should.equal true + + it "should update the active projects metric", -> + @metrics.set.calledWith("editor.active-projects", @project_id).should.equal true + + it "should increment the doc updates", -> + @metrics.inc.calledWith("editor.doc-update").should.equal true describe "unsuccessfully", -> beforeEach -> From ce587a00ba64d3680aee0a505d7d828e72d67540 Mon Sep 17 00:00:00 2001 From: James Allen Date: Mon, 17 Nov 2014 14:35:07 +0000 Subject: [PATCH 023/491] Send web requests with HTTP auth --- services/real-time/app/coffee/WebApiManager.coffee | 4 ++++ services/real-time/config/settings.defaults.coffee | 2 ++ .../real-time/test/unit/coffee/WebApiManagerTests.coffee | 6 ++++++ 3 files changed, 12 insertions(+) diff --git a/services/real-time/app/coffee/WebApiManager.coffee b/services/real-time/app/coffee/WebApiManager.coffee index f18f25c492..63bb6604e0 100644 --- a/services/real-time/app/coffee/WebApiManager.coffee +++ b/services/real-time/app/coffee/WebApiManager.coffee @@ -9,6 +9,10 @@ module.exports = WebApiManager = request.post { url: url qs: {user_id} + auth: + user: settings.apis.web.user + pass: settings.apis.web.pass + sendImmediately: true json: true jar: false }, (error, response, data) -> diff --git a/services/real-time/config/settings.defaults.coffee b/services/real-time/config/settings.defaults.coffee index ccaa02e523..ffc64cabf7 100644 --- a/services/real-time/config/settings.defaults.coffee +++ b/services/real-time/config/settings.defaults.coffee @@ -13,6 +13,8 @@ module.exports = apis: web: url: "http://localhost:3000" + user: "sharelatex" + pass: "password" documentupdater: url: "http://localhost:3003" trackchanges: diff --git a/services/real-time/test/unit/coffee/WebApiManagerTests.coffee b/services/real-time/test/unit/coffee/WebApiManagerTests.coffee index 4cc949dc9c..8ca08547df 100644 --- a/services/real-time/test/unit/coffee/WebApiManagerTests.coffee +++ b/services/real-time/test/unit/coffee/WebApiManagerTests.coffee @@ -15,6 +15,8 @@ describe 'WebApiManager', -> apis: web: url: "http://web.example.com" + user: "username" + pass: "password" "logger-sharelatex": @logger = { log: sinon.stub(), error: sinon.stub() } describe "joinProject", -> @@ -33,6 +35,10 @@ describe 'WebApiManager', -> url: "#{@settings.apis.web.url}/project/#{@project_id}/join" qs: user_id: @user_id + auth: + user: @settings.apis.web.user + pass: @settings.apis.web.pass + sendImmediately: true json: true jar: false }) From 14ace64bc65de43d054e7b1a7f8c8fdde26aec48 Mon Sep 17 00:00:00 2001 From: James Allen Date: Mon, 17 Nov 2014 14:35:32 +0000 Subject: [PATCH 024/491] Ignore grunt forever output --- services/real-time/.gitignore | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/services/real-time/.gitignore b/services/real-time/.gitignore index e3b29a58ff..7c45322029 100644 --- a/services/real-time/.gitignore +++ b/services/real-time/.gitignore @@ -1,5 +1,6 @@ node_modules +forever app.js app/js test/unit/js -test/acceptance/js \ No newline at end of file +test/acceptance/js From 2cb365d2b4f980020d7da407a9eb693578bf896c Mon Sep 17 00:00:00 2001 From: James Allen Date: Mon, 17 Nov 2014 14:38:43 +0000 Subject: [PATCH 025/491] Add /status end point --- services/real-time/app/coffee/Router.coffee | 3 +++ 1 file changed, 3 insertions(+) diff --git a/services/real-time/app/coffee/Router.coffee b/services/real-time/app/coffee/Router.coffee index 14d2f698a8..262239b2a9 100644 --- a/services/real-time/app/coffee/Router.coffee +++ b/services/real-time/app/coffee/Router.coffee @@ -20,6 +20,9 @@ module.exports = Router = app.get "/clients", HttpController.getConnectedClients app.get "/clients/:client_id", HttpController.getConnectedClient + app.get "/status", (req, res, next) -> + res.send "real-time-sharelatex is alive" + session.on 'connection', (error, client, session) -> if error? logger.err err: error, "error when client connected" From cddb4e7279e6c1b9c7128139d2f434935db303cf Mon Sep 17 00:00:00 2001 From: James Allen Date: Mon, 17 Nov 2014 15:01:37 +0000 Subject: [PATCH 026/491] Add cookie parser dependency --- services/real-time/package.json | 1 + 1 file changed, 1 insertion(+) diff --git a/services/real-time/package.json b/services/real-time/package.json index 5f9c323bb3..851c785a64 100644 --- a/services/real-time/package.json +++ b/services/real-time/package.json @@ -10,6 +10,7 @@ "dependencies": { "async": "^0.9.0", "connect-redis": "^2.1.0", + "cookie-parser": "^1.3.3", "express": "^4.10.1", "express-session": "^1.9.1", "logger-sharelatex": "git+https://github.com/sharelatex/logger-sharelatex.git#v1.0.0", From 4d691cf54350ef52a69164e55df1074b5ffa44ef Mon Sep 17 00:00:00 2001 From: James Allen Date: Mon, 17 Nov 2014 15:09:55 +0000 Subject: [PATCH 027/491] Delete client.coffee --- services/real-time/client.coffee | 0 1 file changed, 0 insertions(+), 0 deletions(-) delete mode 100644 services/real-time/client.coffee diff --git a/services/real-time/client.coffee b/services/real-time/client.coffee deleted file mode 100644 index e69de29bb2..0000000000 From 2a05045600e57de99c8b021c0a9e130b82b2bef2 Mon Sep 17 00:00:00 2001 From: James Allen Date: Thu, 20 Nov 2014 16:56:09 +0000 Subject: [PATCH 028/491] Add in redis health check --- services/real-time/app.coffee | 10 ++++++++++ services/real-time/app/coffee/Router.coffee | 3 --- services/real-time/package.json | 2 +- 3 files changed, 11 insertions(+), 4 deletions(-) diff --git a/services/real-time/app.coffee b/services/real-time/app.coffee index b6d8c2e23c..58368683ed 100644 --- a/services/real-time/app.coffee +++ b/services/real-time/app.coffee @@ -38,6 +38,16 @@ io.configure -> #io.enable('browser client gzip') io.set('transports', ['websocket', 'flashsocket', 'htmlfile', 'xhr-polling', 'jsonp-polling']) io.set('log level', 1) + +app.get "/status", (req, res, next) -> + res.send "real-time-sharelatex is alive" + +redisCheck = redis.activeHealthCheckRedis(Settings.redis.web) +app.get "/health_check/redis", (req, res, next) -> + if redisCheck.isAlive() + res.send 200 + else + res.send 500 Router = require "./app/js/Router" Router.configure(app, io, sessionSockets) diff --git a/services/real-time/app/coffee/Router.coffee b/services/real-time/app/coffee/Router.coffee index 262239b2a9..14d2f698a8 100644 --- a/services/real-time/app/coffee/Router.coffee +++ b/services/real-time/app/coffee/Router.coffee @@ -20,9 +20,6 @@ module.exports = Router = app.get "/clients", HttpController.getConnectedClients app.get "/clients/:client_id", HttpController.getConnectedClient - app.get "/status", (req, res, next) -> - res.send "real-time-sharelatex is alive" - session.on 'connection', (error, client, session) -> if error? logger.err err: error, "error when client connected" diff --git a/services/real-time/package.json b/services/real-time/package.json index 851c785a64..6a09d38bba 100644 --- a/services/real-time/package.json +++ b/services/real-time/package.json @@ -15,7 +15,7 @@ "express-session": "^1.9.1", "logger-sharelatex": "git+https://github.com/sharelatex/logger-sharelatex.git#v1.0.0", "metrics-sharelatex": "git+https://github.com/sharelatex/metrics-sharelatex.git#v1.0.0", - "redis-sharelatex": "~0.0.4", + "redis-sharelatex": "0.0.9", "session.socket.io": "^0.1.6", "settings-sharelatex": "git+https://github.com/sharelatex/settings-sharelatex.git#v1.0.0", "socket.io": "0.9.16", From a48c8aad922602845ed1c446dea0ee3e5502f87e Mon Sep 17 00:00:00 2001 From: James Allen Date: Fri, 21 Nov 2014 11:48:59 +0000 Subject: [PATCH 029/491] Support anonymous access --- services/real-time/app/coffee/Router.coffee | 9 +- .../coffee/ClientTrackingTests.coffee | 131 ++++++++++++------ .../acceptance/coffee/SessionTests.coffee | 44 ------ .../coffee/helpers/FixturesManager.coffee | 4 +- 4 files changed, 99 insertions(+), 89 deletions(-) diff --git a/services/real-time/app/coffee/Router.coffee b/services/real-time/app/coffee/Router.coffee index 14d2f698a8..da26d46015 100644 --- a/services/real-time/app/coffee/Router.coffee +++ b/services/real-time/app/coffee/Router.coffee @@ -30,11 +30,10 @@ module.exports = Router = logger.log session: session, client_id: client.id, "client connected" - user = session.user - if !user? or !user._id? - logger.log "terminating session without authenticated user" - client.disconnect() - return + if !session or !session.user? + user = {_id: "anonymous-user"} + else + user = session.user client.on "joinProject", (data = {}, callback) -> WebsocketController.joinProject client, user, data.project_id, (err, args...) -> diff --git a/services/real-time/test/acceptance/coffee/ClientTrackingTests.coffee b/services/real-time/test/acceptance/coffee/ClientTrackingTests.coffee index 90071e61ec..a602df52d2 100644 --- a/services/real-time/test/acceptance/coffee/ClientTrackingTests.coffee +++ b/services/real-time/test/acceptance/coffee/ClientTrackingTests.coffee @@ -9,46 +9,47 @@ FixturesManager = require "./helpers/FixturesManager" async = require "async" describe "clientTracking", -> - before (done) -> - async.series [ - (cb) => - FixturesManager.setUpProject { - privilegeLevel: "owner" - project: { name: "Test Project" } - }, (error, {@user_id, @project_id}) => cb() - - (cb) => - @clientA = RealTimeClient.connect() - @clientA.on "connect", cb - - (cb) => - @clientB = RealTimeClient.connect() - @clientB.on "connect", cb - - (cb) => - @clientA.emit "joinProject", { - project_id: @project_id - }, cb - - (cb) => - @clientB.emit "joinProject", { - project_id: @project_id - }, cb - ], done - describe "when a client updates its cursor location", -> before (done) -> - @updates = [] - @clientB.on "clientTracking.clientUpdated", (data) => - @updates.push data + async.series [ + (cb) => + FixturesManager.setUpProject { + privilegeLevel: "owner" + project: { name: "Test Project" } + }, (error, {@user_id, @project_id}) => cb() - @clientA.emit "clientTracking.updatePosition", { - row: @row = 42 - column: @column = 36 - doc_id: @doc_id = "mock-doc-id" - }, (error) -> - throw error if error? - setTimeout done, 300 # Give the message a chance to reach client B. + (cb) => + @clientA = RealTimeClient.connect() + @clientA.on "connect", cb + + (cb) => + @clientB = RealTimeClient.connect() + + @clientB.on "connect", cb + + (cb) => + @clientA.emit "joinProject", { + project_id: @project_id + }, cb + + (cb) => + @clientB.emit "joinProject", { + project_id: @project_id + }, cb + + (cb) => + @updates = [] + @clientB.on "clientTracking.clientUpdated", (data) => + @updates.push data + + @clientA.emit "clientTracking.updatePosition", { + row: @row = 42 + column: @column = 36 + doc_id: @doc_id = "mock-doc-id" + }, (error) -> + throw error if error? + setTimeout cb, 300 # Give the message a chance to reach client B. + ], done it "should tell other clients about the update", -> @updates.should.deep.equal [ @@ -74,6 +75,58 @@ describe "clientTracking", -> return done() throw new Error("user was never found") + describe "when an anonymous client updates its cursor location", -> + before (done) -> + async.series [ + (cb) => + FixturesManager.setUpProject { + privilegeLevel: "owner" + project: { name: "Test Project" } + publicAccess: "readAndWrite" + }, (error, {@user_id, @project_id}) => cb() - describe "anonymous users", -> - it "should test something..." + (cb) => + @clientA = RealTimeClient.connect() + @clientA.on "connect", cb + (cb) => + @clientA.emit "joinProject", { + project_id: @project_id + }, cb + + (cb) => + RealTimeClient.setSession({}, cb) + + (cb) => + @anonymous = RealTimeClient.connect() + @anonymous.on "connect", cb + + (cb) => + @anonymous.emit "joinProject", { + project_id: @project_id + }, cb + + (cb) => + @updates = [] + @clientA.on "clientTracking.clientUpdated", (data) => + @updates.push data + + @anonymous.emit "clientTracking.updatePosition", { + row: @row = 42 + column: @column = 36 + doc_id: @doc_id = "mock-doc-id" + }, (error) -> + throw error if error? + setTimeout cb, 300 # Give the message a chance to reach client B. + ], done + + it "should tell other clients about the update", -> + @updates.should.deep.equal [ + { + row: @row + column: @column + doc_id: @doc_id + id: @anonymous.socket.sessionid + user_id: "anonymous-user" + name: "Anonymous" + } + ] diff --git a/services/real-time/test/acceptance/coffee/SessionTests.coffee b/services/real-time/test/acceptance/coffee/SessionTests.coffee index 618635db58..fa5378a1f5 100644 --- a/services/real-time/test/acceptance/coffee/SessionTests.coffee +++ b/services/real-time/test/acceptance/coffee/SessionTests.coffee @@ -32,47 +32,3 @@ describe "Session", -> break expect(included).to.equal true done() - - describe "without an established session", -> - before (done) -> - RealTimeClient.unsetSession (error) => - throw error if error? - @client = RealTimeClient.connect() - done() - - it "should get disconnected", (done) -> - @client.on "disconnect", () -> - done() - - it "not should appear in the list of connected clients", (done) -> - RealTimeClient.getConnectedClients (error, clients) => - included = false - for client in clients - if client.client_id == @client.socket.sessionid - included = true - break - expect(included).to.equal false - done() - - describe "without a valid user set on the session", -> - before (done) -> - RealTimeClient.setSession { - foo: "bar" - }, (error) => - throw error if error? - @client = RealTimeClient.connect() - done() - - it "should get disconnected", (done) -> - @client.on "disconnect", () -> - done() - - it "not should appear in the list of connected clients", (done) -> - RealTimeClient.getConnectedClients (error, clients) => - included = false - for client in clients - if client.client_id == @client.socket.sessionid - included = true - break - expect(included).to.equal false - done() \ No newline at end of file diff --git a/services/real-time/test/acceptance/coffee/helpers/FixturesManager.coffee b/services/real-time/test/acceptance/coffee/helpers/FixturesManager.coffee index ad584ce608..9ee5f4ef6f 100644 --- a/services/real-time/test/acceptance/coffee/helpers/FixturesManager.coffee +++ b/services/real-time/test/acceptance/coffee/helpers/FixturesManager.coffee @@ -7,10 +7,12 @@ module.exports = FixturesManager = options.user_id ||= FixturesManager.getRandomId() options.project_id ||= FixturesManager.getRandomId() options.project ||= { name: "Test Project" } - {project_id, user_id, privilegeLevel, project} = options + {project_id, user_id, privilegeLevel, project, publicAccess} = options privileges = {} privileges[user_id] = privilegeLevel + if publicAccess + privileges["anonymous-user"] = publicAccess MockWebServer.createMockProject(project_id, privileges, project) MockWebServer.run (error) => From 57a34e940e31d7a027073811f6d9b3da52a49a62 Mon Sep 17 00:00:00 2001 From: James Allen Date: Mon, 24 Nov 2014 12:05:05 +0000 Subject: [PATCH 030/491] Authorize users before updating their cursor positions --- .../app/coffee/WebsocketController.coffee | 41 ++++++++++--------- .../coffee/WebsocketControllerTests.coffee | 1 + 2 files changed, 23 insertions(+), 19 deletions(-) diff --git a/services/real-time/app/coffee/WebsocketController.coffee b/services/real-time/app/coffee/WebsocketController.coffee index 73f327d22f..add10011b0 100644 --- a/services/real-time/app/coffee/WebsocketController.coffee +++ b/services/real-time/app/coffee/WebsocketController.coffee @@ -113,25 +113,28 @@ module.exports = WebsocketController = ], (error, {project_id, first_name, last_name, email, user_id}) -> return callback(error) if error? logger.log {user_id, project_id, client_id: client.id, cursorData: cursorData}, "updating client position" - cursorData.id = client.id - cursorData.user_id = user_id if user_id? - cursorData.email = email if email? - if first_name? and last_name? - cursorData.name = first_name + " " + last_name - ConnectedUsersManager.updateUserPosition(project_id, client.id, { - first_name: first_name, - last_name: last_name, - email: email, - user_id: user_id - }, { - row: cursorData.row, - column: cursorData.column, - doc_id: cursorData.doc_id - }, callback) - else - cursorData.name = "Anonymous" - callback() - WebsocketLoadBalancer.emitToRoom(project_id, "clientTracking.clientUpdated", cursorData) + + AuthorizationManager.assertClientCanViewProject client, (error) -> + return callback(error) if error? + cursorData.id = client.id + cursorData.user_id = user_id if user_id? + cursorData.email = email if email? + if first_name? and last_name? + cursorData.name = first_name + " " + last_name + ConnectedUsersManager.updateUserPosition(project_id, client.id, { + first_name: first_name, + last_name: last_name, + email: email, + user_id: user_id + }, { + row: cursorData.row, + column: cursorData.column, + doc_id: cursorData.doc_id + }, callback) + else + cursorData.name = "Anonymous" + callback() + WebsocketLoadBalancer.emitToRoom(project_id, "clientTracking.clientUpdated", cursorData) getConnectedUsers: (client, callback = (error, users) ->) -> metrics.inc "editor.get-connected-users" diff --git a/services/real-time/test/unit/coffee/WebsocketControllerTests.coffee b/services/real-time/test/unit/coffee/WebsocketControllerTests.coffee index 5d2e63cc03..ea37cb38ec 100644 --- a/services/real-time/test/unit/coffee/WebsocketControllerTests.coffee +++ b/services/real-time/test/unit/coffee/WebsocketControllerTests.coffee @@ -298,6 +298,7 @@ describe 'WebsocketController', -> beforeEach -> @WebsocketLoadBalancer.emitToRoom = sinon.stub() @ConnectedUsersManager.updateUserPosition = sinon.stub().callsArgWith(4) + @AuthorizationManager.assertClientCanViewProject = sinon.stub().callsArgWith(1, null) @update = { doc_id: @doc_id = "doc-id-123" row: @row = 42 From 0ed7d0c8118235a124fcab9d221a0e611b12d2e2 Mon Sep 17 00:00:00 2001 From: James Allen Date: Mon, 24 Nov 2014 12:09:12 +0000 Subject: [PATCH 031/491] Ignore messages with null room_id --- services/real-time/app/coffee/WebsocketLoadBalancer.coffee | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/services/real-time/app/coffee/WebsocketLoadBalancer.coffee b/services/real-time/app/coffee/WebsocketLoadBalancer.coffee index 966955f117..385bdef222 100644 --- a/services/real-time/app/coffee/WebsocketLoadBalancer.coffee +++ b/services/real-time/app/coffee/WebsocketLoadBalancer.coffee @@ -1,4 +1,5 @@ Settings = require 'settings-sharelatex' +logger = require 'logger-sharelatex' redis = require("redis-sharelatex") rclientPub = redis.createClient(Settings.redis.web) rclientSub = redis.createClient(Settings.redis.web) @@ -8,6 +9,9 @@ module.exports = WebsocketLoadBalancer = rclientSub: rclientSub emitToRoom: (room_id, message, payload...) -> + if !room_id? + logger.err {err: new Error("Empty room_id"), message, payload}, "error emitting to room" + return @rclientPub.publish "editor-events", JSON.stringify room_id: room_id message: message @@ -25,6 +29,6 @@ module.exports = WebsocketLoadBalancer = message = JSON.parse(message) if message.room_id == "all" io.sockets.emit(message.message, message.payload...) - else + else if message.room_id? io.sockets.in(message.room_id).emit(message.message, message.payload...) From 43a008c0bc26927f9cdcb4b19bff3474eeec99d8 Mon Sep 17 00:00:00 2001 From: James Allen Date: Mon, 24 Nov 2014 15:42:13 +0000 Subject: [PATCH 032/491] Stub logger in unit tests --- .../real-time/test/unit/coffee/WebsocketLoadBalancerTests.coffee | 1 + 1 file changed, 1 insertion(+) diff --git a/services/real-time/test/unit/coffee/WebsocketLoadBalancerTests.coffee b/services/real-time/test/unit/coffee/WebsocketLoadBalancerTests.coffee index 547d0aff58..d6add7b3ba 100644 --- a/services/real-time/test/unit/coffee/WebsocketLoadBalancerTests.coffee +++ b/services/real-time/test/unit/coffee/WebsocketLoadBalancerTests.coffee @@ -9,6 +9,7 @@ describe "WebsocketLoadBalancer", -> "redis-sharelatex": createClient: () -> auth:-> + "logger-sharelatex": { log: sinon.stub(), err: sinon.stub() } @io = {} @WebsocketLoadBalancer.rclientPub = publish: sinon.stub() @WebsocketLoadBalancer.rclientSub = From d62dc7ca3a82df512a2124d6eb15bbdb3e1707f6 Mon Sep 17 00:00:00 2001 From: James Allen Date: Mon, 24 Nov 2014 15:42:26 +0000 Subject: [PATCH 033/491] Don't be so verbose with client update errors --- services/real-time/app/coffee/WebsocketController.coffee | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/services/real-time/app/coffee/WebsocketController.coffee b/services/real-time/app/coffee/WebsocketController.coffee index add10011b0..c1037a4ffd 100644 --- a/services/real-time/app/coffee/WebsocketController.coffee +++ b/services/real-time/app/coffee/WebsocketController.coffee @@ -115,7 +115,9 @@ module.exports = WebsocketController = logger.log {user_id, project_id, client_id: client.id, cursorData: cursorData}, "updating client position" AuthorizationManager.assertClientCanViewProject client, (error) -> - return callback(error) if error? + if error? + logger.warn {client_id: client.id, project_id, user_id}, "silently ignoring unauthorized updateClientPosition. Client likely hasn't called joinProject yet." + callback() cursorData.id = client.id cursorData.user_id = user_id if user_id? cursorData.email = email if email? From 7a9f7f0870337716808adcd2b268b5b852b8b71b Mon Sep 17 00:00:00 2001 From: James Allen Date: Mon, 24 Nov 2014 22:28:50 +0000 Subject: [PATCH 034/491] Don't print massive stack trace when there is no room_id --- services/real-time/app/coffee/WebsocketLoadBalancer.coffee | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/services/real-time/app/coffee/WebsocketLoadBalancer.coffee b/services/real-time/app/coffee/WebsocketLoadBalancer.coffee index 385bdef222..3c2c49580c 100644 --- a/services/real-time/app/coffee/WebsocketLoadBalancer.coffee +++ b/services/real-time/app/coffee/WebsocketLoadBalancer.coffee @@ -10,7 +10,7 @@ module.exports = WebsocketLoadBalancer = emitToRoom: (room_id, message, payload...) -> if !room_id? - logger.err {err: new Error("Empty room_id"), message, payload}, "error emitting to room" + logger.warn {message, payload}, "no room_id provided, ignoring emitToRoom" return @rclientPub.publish "editor-events", JSON.stringify room_id: room_id From 99ac814c7d4056a0e67f903e174f3a3979844046 Mon Sep 17 00:00:00 2001 From: James Allen Date: Tue, 25 Nov 2014 09:17:26 +0000 Subject: [PATCH 035/491] Truncate error stack traces to 10 lines --- services/real-time/app.coffee | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/services/real-time/app.coffee b/services/real-time/app.coffee index 58368683ed..2dbf2ebd0e 100644 --- a/services/real-time/app.coffee +++ b/services/real-time/app.coffee @@ -63,4 +63,7 @@ host = Settings.internal.realTime.host server.listen port, host, (error) -> throw error if error? - logger.log "real-time-sharelatex listening on #{host}:#{port}" \ No newline at end of file + logger.log "real-time-sharelatex listening on #{host}:#{port}" + +# Stop huge stack traces in logs from all the socket.io parsing steps. +Error.stackTraceLimit = 10 \ No newline at end of file From 79cd0e6a5cf2ea7f3b4d37d782f915e50b2993a2 Mon Sep 17 00:00:00 2001 From: James Allen Date: Thu, 5 Feb 2015 13:41:31 +0000 Subject: [PATCH 036/491] Record user id correctly when updating position --- services/real-time/app/coffee/WebsocketController.coffee | 2 +- .../real-time/test/unit/coffee/WebsocketControllerTests.coffee | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/services/real-time/app/coffee/WebsocketController.coffee b/services/real-time/app/coffee/WebsocketController.coffee index c1037a4ffd..d578c36622 100644 --- a/services/real-time/app/coffee/WebsocketController.coffee +++ b/services/real-time/app/coffee/WebsocketController.coffee @@ -127,7 +127,7 @@ module.exports = WebsocketController = first_name: first_name, last_name: last_name, email: email, - user_id: user_id + _id: user_id }, { row: cursorData.row, column: cursorData.column, diff --git a/services/real-time/test/unit/coffee/WebsocketControllerTests.coffee b/services/real-time/test/unit/coffee/WebsocketControllerTests.coffee index ea37cb38ec..2f9e66d1ef 100644 --- a/services/real-time/test/unit/coffee/WebsocketControllerTests.coffee +++ b/services/real-time/test/unit/coffee/WebsocketControllerTests.coffee @@ -331,7 +331,7 @@ describe 'WebsocketController', -> it "should send the cursor data to the connected user manager", (done)-> @ConnectedUsersManager.updateUserPosition.calledWith(@project_id, @client.id, { - user_id: @user_id, + _id: @user_id, email: @email, first_name: @first_name, last_name: @last_name From f72fa9de34d4d08284d75b9f3498041881ca8ff5 Mon Sep 17 00:00:00 2001 From: David Renshaw Date: Mon, 9 Feb 2015 17:34:01 -0500 Subject: [PATCH 037/491] 'request' is a production dependency --- services/real-time/package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/services/real-time/package.json b/services/real-time/package.json index 6a09d38bba..bca24ffa90 100644 --- a/services/real-time/package.json +++ b/services/real-time/package.json @@ -16,6 +16,7 @@ "logger-sharelatex": "git+https://github.com/sharelatex/logger-sharelatex.git#v1.0.0", "metrics-sharelatex": "git+https://github.com/sharelatex/metrics-sharelatex.git#v1.0.0", "redis-sharelatex": "0.0.9", + "request": "~2.34.0", "session.socket.io": "^0.1.6", "settings-sharelatex": "git+https://github.com/sharelatex/settings-sharelatex.git#v1.0.0", "socket.io": "0.9.16", @@ -33,7 +34,6 @@ "grunt-forever": "~0.4.4", "grunt-mocha-test": "~0.10.2", "grunt-shell": "~0.7.0", - "request": "~2.34.0", "sandboxed-module": "~0.3.0", "sinon": "~1.5.2", "uid-safe": "^1.0.1", From f0462f0b1f989dfddfede6f19e8ec687d5ac2e46 Mon Sep 17 00:00:00 2001 From: James Allen Date: Tue, 10 Feb 2015 13:14:13 +0000 Subject: [PATCH 038/491] Bump version to 0.1.2 --- services/real-time/package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/services/real-time/package.json b/services/real-time/package.json index bca24ffa90..75c61eafa8 100644 --- a/services/real-time/package.json +++ b/services/real-time/package.json @@ -1,6 +1,6 @@ { "name": "real-time-sharelatex", - "version": "0.0.1", + "version": "0.1.2", "description": "The socket.io layer of ShareLaTeX for real-time editor interactions", "author": "ShareLaTeX ", "repository": { From 5ee71a423da1abe5ba70778f45adbbbd63e6a4b1 Mon Sep 17 00:00:00 2001 From: James Allen Date: Thu, 26 Feb 2015 11:23:01 +0000 Subject: [PATCH 039/491] Release version 0.1.3 --- services/real-time/package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/services/real-time/package.json b/services/real-time/package.json index 75c61eafa8..56a13842a9 100644 --- a/services/real-time/package.json +++ b/services/real-time/package.json @@ -1,6 +1,6 @@ { "name": "real-time-sharelatex", - "version": "0.1.2", + "version": "0.1.3", "description": "The socket.io layer of ShareLaTeX for real-time editor interactions", "author": "ShareLaTeX ", "repository": { From 536118b0cfc8bfe1eeded8fd42b08abb1f8f73a2 Mon Sep 17 00:00:00 2001 From: James Allen Date: Tue, 3 Mar 2015 17:15:19 +0000 Subject: [PATCH 040/491] Relay messages received via HTTP into the project --- .../real-time/app/coffee/HttpApiController.coffee | 8 ++++++++ services/real-time/app/coffee/Router.coffee | 12 ++++++++++++ .../app/coffee/WebsocketLoadBalancer.coffee | 1 + services/real-time/config/settings.defaults.coffee | 2 ++ services/real-time/package.json | 2 ++ 5 files changed, 25 insertions(+) create mode 100644 services/real-time/app/coffee/HttpApiController.coffee diff --git a/services/real-time/app/coffee/HttpApiController.coffee b/services/real-time/app/coffee/HttpApiController.coffee new file mode 100644 index 0000000000..faa06b9822 --- /dev/null +++ b/services/real-time/app/coffee/HttpApiController.coffee @@ -0,0 +1,8 @@ +WebsocketLoadBalancer = require "./WebsocketLoadBalancer" +logger = require "logger-sharelatex" + +module.exports = HttpApiController = + sendMessage: (req, res, next) -> + logger.log {message: req.params.message}, "sending message" + WebsocketLoadBalancer.emitToRoom req.params.project_id, req.params.message, req.body + res.send 204 # No content \ No newline at end of file diff --git a/services/real-time/app/coffee/Router.coffee b/services/real-time/app/coffee/Router.coffee index da26d46015..0fee1cb837 100644 --- a/services/real-time/app/coffee/Router.coffee +++ b/services/real-time/app/coffee/Router.coffee @@ -1,8 +1,18 @@ metrics = require "metrics-sharelatex" logger = require "logger-sharelatex" +settings = require "settings-sharelatex" WebsocketController = require "./WebsocketController" HttpController = require "./HttpController" +HttpApiController = require "./HttpApiController" Utils = require "./Utils" +bodyParser = require "body-parser" + +basicAuth = require('basic-auth-connect') +httpAuth = basicAuth (user, pass)-> + isValid = user == settings.internal.realTime.user and pass == settings.internal.realTime.pass + if !isValid + logger.err user:user, pass:pass, "invalid login details" + return isValid module.exports = Router = _handleError: (callback = ((error) ->), error, client, method, extraAttrs = {}) -> @@ -19,6 +29,8 @@ module.exports = Router = app.set("io", io) app.get "/clients", HttpController.getConnectedClients app.get "/clients/:client_id", HttpController.getConnectedClient + + app.post "/project/:project_id/message/:message", httpAuth, bodyParser.json(limit: "5mb"), HttpApiController.sendMessage session.on 'connection', (error, client, session) -> if error? diff --git a/services/real-time/app/coffee/WebsocketLoadBalancer.coffee b/services/real-time/app/coffee/WebsocketLoadBalancer.coffee index 3c2c49580c..38a1029d83 100644 --- a/services/real-time/app/coffee/WebsocketLoadBalancer.coffee +++ b/services/real-time/app/coffee/WebsocketLoadBalancer.coffee @@ -12,6 +12,7 @@ module.exports = WebsocketLoadBalancer = if !room_id? logger.warn {message, payload}, "no room_id provided, ignoring emitToRoom" return + logger.log {room_id, message, payload}, "emitting to room" @rclientPub.publish "editor-events", JSON.stringify room_id: room_id message: message diff --git a/services/real-time/config/settings.defaults.coffee b/services/real-time/config/settings.defaults.coffee index ffc64cabf7..50abd714c4 100644 --- a/services/real-time/config/settings.defaults.coffee +++ b/services/real-time/config/settings.defaults.coffee @@ -9,6 +9,8 @@ module.exports = realTime: port: 3026 host: "localhost" + user: "sharelatex" + pass: "password" apis: web: diff --git a/services/real-time/package.json b/services/real-time/package.json index 56a13842a9..648e57abed 100644 --- a/services/real-time/package.json +++ b/services/real-time/package.json @@ -9,6 +9,8 @@ }, "dependencies": { "async": "^0.9.0", + "basic-auth-connect": "^1.0.0", + "body-parser": "^1.12.0", "connect-redis": "^2.1.0", "cookie-parser": "^1.3.3", "express": "^4.10.1", From 5da5c5c4354e9f5e75a4ad3c929c3033425d62c3 Mon Sep 17 00:00:00 2001 From: James Allen Date: Thu, 12 Mar 2015 14:32:35 +0000 Subject: [PATCH 041/491] Accept arrays of messages to send to client --- services/real-time/app/coffee/HttpApiController.coffee | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/services/real-time/app/coffee/HttpApiController.coffee b/services/real-time/app/coffee/HttpApiController.coffee index faa06b9822..1082a11b34 100644 --- a/services/real-time/app/coffee/HttpApiController.coffee +++ b/services/real-time/app/coffee/HttpApiController.coffee @@ -4,5 +4,9 @@ logger = require "logger-sharelatex" module.exports = HttpApiController = sendMessage: (req, res, next) -> logger.log {message: req.params.message}, "sending message" - WebsocketLoadBalancer.emitToRoom req.params.project_id, req.params.message, req.body + if Array.isArray(req.body) + for payload in req.body + WebsocketLoadBalancer.emitToRoom req.params.project_id, req.params.message, payload + else + WebsocketLoadBalancer.emitToRoom req.params.project_id, req.params.message, req.body res.send 204 # No content \ No newline at end of file From 985abe42feded566796af8763e141111991569fd Mon Sep 17 00:00:00 2001 From: James Allen Date: Fri, 20 Mar 2015 14:21:17 +0000 Subject: [PATCH 042/491] Release version 0.1.4 --- services/real-time/package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/services/real-time/package.json b/services/real-time/package.json index 648e57abed..ed405612bd 100644 --- a/services/real-time/package.json +++ b/services/real-time/package.json @@ -1,6 +1,6 @@ { "name": "real-time-sharelatex", - "version": "0.1.3", + "version": "0.1.4", "description": "The socket.io layer of ShareLaTeX for real-time editor interactions", "author": "ShareLaTeX ", "repository": { From 42e7d5d4b68a16ac6f19ff77164510e14cb2f142 Mon Sep 17 00:00:00 2001 From: Henry Oswald Date: Thu, 30 Apr 2015 15:05:31 +0100 Subject: [PATCH 043/491] make startup message consistent --- services/real-time/app.coffee | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/services/real-time/app.coffee b/services/real-time/app.coffee index 2dbf2ebd0e..0b9d12584f 100644 --- a/services/real-time/app.coffee +++ b/services/real-time/app.coffee @@ -63,7 +63,7 @@ host = Settings.internal.realTime.host server.listen port, host, (error) -> throw error if error? - logger.log "real-time-sharelatex listening on #{host}:#{port}" + logger.info "realtime starting up, listening on #{host}:#{port}" # Stop huge stack traces in logs from all the socket.io parsing steps. Error.stackTraceLimit = 10 \ No newline at end of file From d79793c34fac16b2fb7dbb1e567be1308f987cdb Mon Sep 17 00:00:00 2001 From: James Allen Date: Sat, 29 Aug 2015 08:22:43 +0100 Subject: [PATCH 044/491] Monitor get document times --- services/real-time/app/coffee/DocumentUpdaterManager.coffee | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/services/real-time/app/coffee/DocumentUpdaterManager.coffee b/services/real-time/app/coffee/DocumentUpdaterManager.coffee index 8610b454ec..1bc9fd3bc9 100644 --- a/services/real-time/app/coffee/DocumentUpdaterManager.coffee +++ b/services/real-time/app/coffee/DocumentUpdaterManager.coffee @@ -7,11 +7,11 @@ rclient = redis.createClient(settings.redis.web) module.exports = DocumentUpdaterManager = getDocument: (project_id, doc_id, fromVersion, callback = (error, exists, doclines, version) ->) -> - #timer = new metrics.Timer("get-document") + timer = new metrics.Timer("get-document") url = "#{settings.apis.documentupdater.url}/project/#{project_id}/doc/#{doc_id}?fromVersion=#{fromVersion}" logger.log {project_id, doc_id, fromVersion}, "getting doc from document updater" request.get url, (err, res, body) -> - #timer.done() + timer.done() if err? logger.error {err, url, project_id, doc_id}, "error getting doc from doc updater" return callback(err) @@ -55,4 +55,4 @@ module.exports = DocumentUpdaterManager = multi.rpush "pending-updates-list", doc_key multi.exec (error) -> return callback(error) if error? - callback() \ No newline at end of file + callback() From 3c70acd560a100d5cccb17ad49b7e3bf591ea359 Mon Sep 17 00:00:00 2001 From: Henry Oswald Date: Sat, 29 Aug 2015 08:26:01 +0100 Subject: [PATCH 045/491] added metrics into realtime around doc update manager --- services/real-time/app/coffee/DocumentUpdaterManager.coffee | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/services/real-time/app/coffee/DocumentUpdaterManager.coffee b/services/real-time/app/coffee/DocumentUpdaterManager.coffee index 1bc9fd3bc9..a2c9fe54f8 100644 --- a/services/real-time/app/coffee/DocumentUpdaterManager.coffee +++ b/services/real-time/app/coffee/DocumentUpdaterManager.coffee @@ -1,6 +1,7 @@ request = require "request" logger = require "logger-sharelatex" settings = require "settings-sharelatex" +metrics = require("metrics-sharelatex") redis = require("redis-sharelatex") rclient = redis.createClient(settings.redis.web) @@ -30,10 +31,10 @@ module.exports = DocumentUpdaterManager = flushProjectToMongoAndDelete: (project_id, callback = ()->) -> logger.log project_id:project_id, "deleting project from document updater" - #timer = new metrics.Timer("delete.mongo.project") + timer = new metrics.Timer("delete.mongo.project") url = "#{settings.apis.documentupdater.url}/project/#{project_id}" request.del url, (err, res, body)-> - #timer.done() + timer.done() if err? logger.error {err, project_id}, "error deleting project from document updater" return callback(err) From da28b0fc707e0a900445dba442cb299bfaf1e90a Mon Sep 17 00:00:00 2001 From: James Allen Date: Mon, 31 Aug 2015 14:04:54 +0100 Subject: [PATCH 046/491] Use updated metrics with unlimited socket config --- services/real-time/package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/services/real-time/package.json b/services/real-time/package.json index ed405612bd..043d3524d5 100644 --- a/services/real-time/package.json +++ b/services/real-time/package.json @@ -16,7 +16,7 @@ "express": "^4.10.1", "express-session": "^1.9.1", "logger-sharelatex": "git+https://github.com/sharelatex/logger-sharelatex.git#v1.0.0", - "metrics-sharelatex": "git+https://github.com/sharelatex/metrics-sharelatex.git#v1.0.0", + "metrics-sharelatex": "git+https://github.com/sharelatex/metrics-sharelatex.git#v1.3.0", "redis-sharelatex": "0.0.9", "request": "~2.34.0", "session.socket.io": "^0.1.6", From 503b766dccbf15ea3657a22acc8f7c068726fe69 Mon Sep 17 00:00:00 2001 From: James Allen Date: Thu, 19 Nov 2015 10:58:28 +0000 Subject: [PATCH 047/491] For duplicate ops only send ack to submitting client When a duplicate op is received, we only need to ack it to client that sent it. Only that client is having trouble, and all other clients will already have received it. --- .../coffee/DocumentUpdaterController.coffee | 2 +- .../DocumentUpdaterControllerTests.coffee | 49 +++++++++++++------ 2 files changed, 35 insertions(+), 16 deletions(-) diff --git a/services/real-time/app/coffee/DocumentUpdaterController.coffee b/services/real-time/app/coffee/DocumentUpdaterController.coffee index 3de0becfc9..e74b01259b 100644 --- a/services/real-time/app/coffee/DocumentUpdaterController.coffee +++ b/services/real-time/app/coffee/DocumentUpdaterController.coffee @@ -24,7 +24,7 @@ module.exports = DocumentUpdaterController = if client.id == update.meta.source logger.log doc_id: doc_id, version: update.v, source: update.meta?.source, "distributing update to sender" client.emit "otUpdateApplied", v: update.v, doc: update.doc - else + else if !update.dup # Duplicate ops should just be sent back to sending client for acknowledgement logger.log doc_id: doc_id, version: update.v, source: update.meta?.source, client_id: client.id, "distributing update to collaborator" client.emit "otUpdateApplied", update diff --git a/services/real-time/test/unit/coffee/DocumentUpdaterControllerTests.coffee b/services/real-time/test/unit/coffee/DocumentUpdaterControllerTests.coffee index 479231743c..186b493669 100644 --- a/services/real-time/test/unit/coffee/DocumentUpdaterControllerTests.coffee +++ b/services/real-time/test/unit/coffee/DocumentUpdaterControllerTests.coffee @@ -68,24 +68,43 @@ describe "DocumentUpdaterController", -> doc: @doc_id @io.sockets = clients: sinon.stub().returns([@sourceClient, @otherClients...]) - @EditorUpdatesController._applyUpdateFromDocumentUpdater @io, @doc_id, @update + + describe "normally", -> + beforeEach -> + @EditorUpdatesController._applyUpdateFromDocumentUpdater @io, @doc_id, @update - it "should send a version bump to the source client", -> - @sourceClient.emit - .calledWith("otUpdateApplied", v: @version, doc: @doc_id) - .should.equal true - - it "should get the clients connected to the document", -> - @io.sockets.clients - .calledWith(@doc_id) - .should.equal true - - it "should send the full update to the other clients", -> - for client in @otherClients - client.emit - .calledWith("otUpdateApplied", @update) + it "should send a version bump to the source client", -> + @sourceClient.emit + .calledWith("otUpdateApplied", v: @version, doc: @doc_id) .should.equal true + it "should get the clients connected to the document", -> + @io.sockets.clients + .calledWith(@doc_id) + .should.equal true + + it "should send the full update to the other clients", -> + for client in @otherClients + client.emit + .calledWith("otUpdateApplied", @update) + .should.equal true + + describe "with a duplicate op", -> + beforeEach -> + @update.dup = true + @EditorUpdatesController._applyUpdateFromDocumentUpdater @io, @doc_id, @update + + it "should send a version bump to the source client as usual", -> + @sourceClient.emit + .calledWith("otUpdateApplied", v: @version, doc: @doc_id) + .should.equal true + + it "should not send anything to the other clients (they've already had the op)", -> + for client in @otherClients + client.emit + .calledWith("otUpdateApplied") + .should.equal false + describe "_processErrorFromDocumentUpdater", -> beforeEach -> @clients = [new MockClient(), new MockClient()] From 0372fa32035cb03114764c3ddee8064b86d95c52 Mon Sep 17 00:00:00 2001 From: James Allen Date: Mon, 30 Nov 2015 15:25:09 +0000 Subject: [PATCH 048/491] Add in extra logging about size of messages --- .../real-time/app/coffee/DocumentUpdaterController.coffee | 4 ++++ services/real-time/app/coffee/WebsocketLoadBalancer.coffee | 5 +++-- 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/services/real-time/app/coffee/DocumentUpdaterController.coffee b/services/real-time/app/coffee/DocumentUpdaterController.coffee index e74b01259b..441cff70ce 100644 --- a/services/real-time/app/coffee/DocumentUpdaterController.coffee +++ b/services/real-time/app/coffee/DocumentUpdaterController.coffee @@ -3,6 +3,8 @@ settings = require 'settings-sharelatex' redis = require("redis-sharelatex") rclient = redis.createClient(settings.redis.web) +MESSAGE_SIZE_LOG_LIMIT = 1024 * 1024 * 1024 # 1Mb + module.exports = DocumentUpdaterController = # DocumentUpdaterController is responsible for updates that come via Redis # Pub/Sub from the document updater. @@ -13,6 +15,8 @@ module.exports = DocumentUpdaterController = DocumentUpdaterController._processMessageFromDocumentUpdater(io, channel, message) _processMessageFromDocumentUpdater: (io, channel, message) -> + if message.length > MESSAGE_SIZE_LOG_LIMIT + logger.log {length: message.length, head: message.slice(0,200)}, "large message from doc updater" message = JSON.parse message if message.op? DocumentUpdaterController._applyUpdateFromDocumentUpdater(io, message.doc_id, message.op) diff --git a/services/real-time/app/coffee/WebsocketLoadBalancer.coffee b/services/real-time/app/coffee/WebsocketLoadBalancer.coffee index 38a1029d83..4635a42256 100644 --- a/services/real-time/app/coffee/WebsocketLoadBalancer.coffee +++ b/services/real-time/app/coffee/WebsocketLoadBalancer.coffee @@ -12,11 +12,12 @@ module.exports = WebsocketLoadBalancer = if !room_id? logger.warn {message, payload}, "no room_id provided, ignoring emitToRoom" return - logger.log {room_id, message, payload}, "emitting to room" - @rclientPub.publish "editor-events", JSON.stringify + data = JSON.stringify room_id: room_id message: message payload: payload + logger.log {room_id, message, payload, length: data.length}, "emitting to room" + @rclientPub.publish "editor-events", data emitToAll: (message, payload...) -> @emitToRoom "all", message, payload... From 92d18d7e2e7fd8bb1eb856b48a8f754ace8d930e Mon Sep 17 00:00:00 2001 From: James Allen Date: Mon, 30 Nov 2015 15:40:03 +0000 Subject: [PATCH 049/491] Reduce limit to actuall 1Mb, not 1Gb --- services/real-time/app/coffee/DocumentUpdaterController.coffee | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/services/real-time/app/coffee/DocumentUpdaterController.coffee b/services/real-time/app/coffee/DocumentUpdaterController.coffee index 441cff70ce..3123aee942 100644 --- a/services/real-time/app/coffee/DocumentUpdaterController.coffee +++ b/services/real-time/app/coffee/DocumentUpdaterController.coffee @@ -3,7 +3,7 @@ settings = require 'settings-sharelatex' redis = require("redis-sharelatex") rclient = redis.createClient(settings.redis.web) -MESSAGE_SIZE_LOG_LIMIT = 1024 * 1024 * 1024 # 1Mb +MESSAGE_SIZE_LOG_LIMIT = 1024 * 1024 # 1Mb module.exports = DocumentUpdaterController = # DocumentUpdaterController is responsible for updates that come via Redis From 830d676f4f443a3b09d1df33ff8879a8e1eab593 Mon Sep 17 00:00:00 2001 From: James Allen Date: Tue, 1 Dec 2015 11:05:49 +0000 Subject: [PATCH 050/491] Add in limit on all JSON parsing --- .../coffee/DocumentUpdaterController.coffee | 16 +++++---- .../real-time/app/coffee/SafeJsonParse.coffee | 13 +++++++ .../app/coffee/WebsocketLoadBalancer.coffee | 14 +++++--- .../real-time/config/settings.defaults.coffee | 4 ++- .../DocumentUpdaterControllerTests.coffee | 10 ++++++ .../test/unit/coffee/SafeJsonParseTest.coffee | 34 +++++++++++++++++++ .../coffee/WebsocketLoadBalancerTests.coffee | 12 ++++++- 7 files changed, 89 insertions(+), 14 deletions(-) create mode 100644 services/real-time/app/coffee/SafeJsonParse.coffee create mode 100644 services/real-time/test/unit/coffee/SafeJsonParseTest.coffee diff --git a/services/real-time/app/coffee/DocumentUpdaterController.coffee b/services/real-time/app/coffee/DocumentUpdaterController.coffee index 3123aee942..9ff6b8ee62 100644 --- a/services/real-time/app/coffee/DocumentUpdaterController.coffee +++ b/services/real-time/app/coffee/DocumentUpdaterController.coffee @@ -2,6 +2,7 @@ logger = require "logger-sharelatex" settings = require 'settings-sharelatex' redis = require("redis-sharelatex") rclient = redis.createClient(settings.redis.web) +SafeJsonParse = require "./SafeJsonParse" MESSAGE_SIZE_LOG_LIMIT = 1024 * 1024 # 1Mb @@ -15,13 +16,14 @@ module.exports = DocumentUpdaterController = DocumentUpdaterController._processMessageFromDocumentUpdater(io, channel, message) _processMessageFromDocumentUpdater: (io, channel, message) -> - if message.length > MESSAGE_SIZE_LOG_LIMIT - logger.log {length: message.length, head: message.slice(0,200)}, "large message from doc updater" - message = JSON.parse message - if message.op? - DocumentUpdaterController._applyUpdateFromDocumentUpdater(io, message.doc_id, message.op) - else if message.error? - DocumentUpdaterController._processErrorFromDocumentUpdater(io, message.doc_id, message.error, message) + SafeJsonParse.parse message, (error, message) -> + if error? + logger.error {err: error, channel}, "error parsing JSON" + return + if message.op? + DocumentUpdaterController._applyUpdateFromDocumentUpdater(io, message.doc_id, message.op) + else if message.error? + DocumentUpdaterController._processErrorFromDocumentUpdater(io, message.doc_id, message.error, message) _applyUpdateFromDocumentUpdater: (io, doc_id, update) -> for client in io.sockets.clients(doc_id) diff --git a/services/real-time/app/coffee/SafeJsonParse.coffee b/services/real-time/app/coffee/SafeJsonParse.coffee new file mode 100644 index 0000000000..edfa921d74 --- /dev/null +++ b/services/real-time/app/coffee/SafeJsonParse.coffee @@ -0,0 +1,13 @@ +Settings = require "settings-sharelatex" +logger = require "logger-sharelatex" + +module.exports = + parse: (data, callback = (error, parsed) ->) -> + if data.length > (Settings.max_doc_length or 2 * 1024 * 1024) + logger.error {head: data.slice(0,1024)}, "data too large to parse" + return callback new Error("data too large to parse") + try + parsed = JSON.parse(data) + catch e + return callback e + callback null, parsed \ No newline at end of file diff --git a/services/real-time/app/coffee/WebsocketLoadBalancer.coffee b/services/real-time/app/coffee/WebsocketLoadBalancer.coffee index 4635a42256..d346c41107 100644 --- a/services/real-time/app/coffee/WebsocketLoadBalancer.coffee +++ b/services/real-time/app/coffee/WebsocketLoadBalancer.coffee @@ -1,6 +1,7 @@ Settings = require 'settings-sharelatex' logger = require 'logger-sharelatex' redis = require("redis-sharelatex") +SafeJsonParse = require "./SafeJsonParse" rclientPub = redis.createClient(Settings.redis.web) rclientSub = redis.createClient(Settings.redis.web) @@ -28,9 +29,12 @@ module.exports = WebsocketLoadBalancer = WebsocketLoadBalancer._processEditorEvent io, channel, message _processEditorEvent: (io, channel, message) -> - message = JSON.parse(message) - if message.room_id == "all" - io.sockets.emit(message.message, message.payload...) - else if message.room_id? - io.sockets.in(message.room_id).emit(message.message, message.payload...) + SafeJsonParse.parse message, (error, message) -> + if error? + logger.error {err: error, channel}, "error parsing JSON" + return + if message.room_id == "all" + io.sockets.emit(message.message, message.payload...) + else if message.room_id? + io.sockets.in(message.room_id).emit(message.message, message.payload...) diff --git a/services/real-time/config/settings.defaults.coffee b/services/real-time/config/settings.defaults.coffee index 50abd714c4..eaf5e7d61e 100644 --- a/services/real-time/config/settings.defaults.coffee +++ b/services/real-time/config/settings.defaults.coffee @@ -25,4 +25,6 @@ module.exports = security: sessionSecret: "secret-please-change" - cookieName:"sharelatex.sid" \ No newline at end of file + cookieName:"sharelatex.sid" + + max_doc_length: 2 * 1024 * 1024 # 2mb \ No newline at end of file diff --git a/services/real-time/test/unit/coffee/DocumentUpdaterControllerTests.coffee b/services/real-time/test/unit/coffee/DocumentUpdaterControllerTests.coffee index 186b493669..8673b12b41 100644 --- a/services/real-time/test/unit/coffee/DocumentUpdaterControllerTests.coffee +++ b/services/real-time/test/unit/coffee/DocumentUpdaterControllerTests.coffee @@ -17,6 +17,8 @@ describe "DocumentUpdaterController", -> "redis-sharelatex" : createClient: ()=> @rclient = {auth:->} + "./SafeJsonParse": @SafeJsonParse = + parse: (data, cb) => cb null, JSON.parse(data) describe "listenForUpdatesFromDocumentUpdater", -> beforeEach -> @@ -31,6 +33,14 @@ describe "DocumentUpdaterController", -> @rclient.on.calledWith("message").should.equal true describe "_processMessageFromDocumentUpdater", -> + describe "with bad JSON", -> + beforeEach -> + @SafeJsonParse.parse = sinon.stub().callsArgWith 1, new Error("oops") + @EditorUpdatesController._processMessageFromDocumentUpdater @io, "applied-ops", "blah" + + it "should log an error", -> + @logger.error.called.should.equal true + describe "with update", -> beforeEach -> @message = diff --git a/services/real-time/test/unit/coffee/SafeJsonParseTest.coffee b/services/real-time/test/unit/coffee/SafeJsonParseTest.coffee new file mode 100644 index 0000000000..6a6b5a951c --- /dev/null +++ b/services/real-time/test/unit/coffee/SafeJsonParseTest.coffee @@ -0,0 +1,34 @@ +require('chai').should() +expect = require("chai").expect +SandboxedModule = require('sandboxed-module') +modulePath = '../../../app/js/SafeJsonParse' +sinon = require("sinon") + +describe 'SafeJsonParse', -> + beforeEach -> + @SafeJsonParse = SandboxedModule.require modulePath, requires: + "settings-sharelatex": @Settings = { + max_doc_length: 16 * 1024 + } + "logger-sharelatex": @logger = {error: sinon.stub()} + + describe "parse", -> + it "should parse documents correctly", (done) -> + @SafeJsonParse.parse '{"foo": "bar"}', (error, parsed) -> + expect(parsed).to.deep.equal {foo: "bar"} + done() + + it "should return an error on bad data", (done) -> + @SafeJsonParse.parse 'blah', (error, parsed) -> + expect(error).to.exist + done() + + it "should return an error on oversized data", (done) -> + # we have a 2k overhead on top of max size + big_blob = Array(16*1024).join("A") + data = "{\"foo\": \"#{big_blob}\"}" + @Settings.max_doc_length = 2 * 1024 + @SafeJsonParse.parse data, (error, parsed) => + @logger.error.called.should.equal true + expect(error).to.exist + done() \ No newline at end of file diff --git a/services/real-time/test/unit/coffee/WebsocketLoadBalancerTests.coffee b/services/real-time/test/unit/coffee/WebsocketLoadBalancerTests.coffee index d6add7b3ba..07afe988ab 100644 --- a/services/real-time/test/unit/coffee/WebsocketLoadBalancerTests.coffee +++ b/services/real-time/test/unit/coffee/WebsocketLoadBalancerTests.coffee @@ -9,7 +9,9 @@ describe "WebsocketLoadBalancer", -> "redis-sharelatex": createClient: () -> auth:-> - "logger-sharelatex": { log: sinon.stub(), err: sinon.stub() } + "logger-sharelatex": @logger = { log: sinon.stub(), error: sinon.stub() } + "./SafeJsonParse": @SafeJsonParse = + parse: (data, cb) => cb null, JSON.parse(data) @io = {} @WebsocketLoadBalancer.rclientPub = publish: sinon.stub() @WebsocketLoadBalancer.rclientSub = @@ -59,6 +61,14 @@ describe "WebsocketLoadBalancer", -> .should.equal true describe "_processEditorEvent", -> + describe "with bad JSON", -> + beforeEach -> + @SafeJsonParse.parse = sinon.stub().callsArgWith 1, new Error("oops") + @WebsocketLoadBalancer._processEditorEvent(@io, "editor-events", "blah") + + it "should log an error", -> + @logger.error.called.should.equal true + describe "with a designated room", -> beforeEach -> @io.sockets = From 3580e3ba6ba112b8dbb79b5b43855955bf9c0acb Mon Sep 17 00:00:00 2001 From: James Allen Date: Tue, 1 Dec 2015 17:18:45 +0000 Subject: [PATCH 051/491] Update to latest logger --- services/real-time/package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/services/real-time/package.json b/services/real-time/package.json index 043d3524d5..539c3ff14e 100644 --- a/services/real-time/package.json +++ b/services/real-time/package.json @@ -15,7 +15,7 @@ "cookie-parser": "^1.3.3", "express": "^4.10.1", "express-session": "^1.9.1", - "logger-sharelatex": "git+https://github.com/sharelatex/logger-sharelatex.git#v1.0.0", + "logger-sharelatex": "git+https://github.com/sharelatex/logger-sharelatex.git#master", "metrics-sharelatex": "git+https://github.com/sharelatex/metrics-sharelatex.git#v1.3.0", "redis-sharelatex": "0.0.9", "request": "~2.34.0", From 5b17764da2cb8f0b4604dca1c9b52b586ff00c65 Mon Sep 17 00:00:00 2001 From: James Allen Date: Tue, 1 Dec 2015 17:28:26 +0000 Subject: [PATCH 052/491] Pin down logger version number --- services/real-time/package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/services/real-time/package.json b/services/real-time/package.json index 539c3ff14e..d56328899b 100644 --- a/services/real-time/package.json +++ b/services/real-time/package.json @@ -15,7 +15,7 @@ "cookie-parser": "^1.3.3", "express": "^4.10.1", "express-session": "^1.9.1", - "logger-sharelatex": "git+https://github.com/sharelatex/logger-sharelatex.git#master", + "logger-sharelatex": "git+https://github.com/sharelatex/logger-sharelatex.git#v1.1.0", "metrics-sharelatex": "git+https://github.com/sharelatex/metrics-sharelatex.git#v1.3.0", "redis-sharelatex": "0.0.9", "request": "~2.34.0", From 73cd1a3e92cf85c84583665ba27416f0924d9cec Mon Sep 17 00:00:00 2001 From: James Allen Date: Thu, 3 Dec 2015 16:50:56 +0000 Subject: [PATCH 053/491] Use latest version of metrics with suitable event loop monitoring --- services/real-time/app.coffee | 1 + services/real-time/package.json | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/services/real-time/app.coffee b/services/real-time/app.coffee index 0b9d12584f..dff0fd17bd 100644 --- a/services/real-time/app.coffee +++ b/services/real-time/app.coffee @@ -12,6 +12,7 @@ logger.initialize("real-time-sharelatex") Metrics = require("metrics-sharelatex") Metrics.initialize("real-time") +Metrics.event_loop.monitor(logger) rclient = redis.createClient(Settings.redis.web) diff --git a/services/real-time/package.json b/services/real-time/package.json index d56328899b..b0c7b46c2d 100644 --- a/services/real-time/package.json +++ b/services/real-time/package.json @@ -16,7 +16,7 @@ "express": "^4.10.1", "express-session": "^1.9.1", "logger-sharelatex": "git+https://github.com/sharelatex/logger-sharelatex.git#v1.1.0", - "metrics-sharelatex": "git+https://github.com/sharelatex/metrics-sharelatex.git#v1.3.0", + "metrics-sharelatex": "git+https://github.com/sharelatex/metrics-sharelatex.git#v1.4.0", "redis-sharelatex": "0.0.9", "request": "~2.34.0", "session.socket.io": "^0.1.6", From b28e5ac6b297e6d137697ab6f7f7eade2dd06f94 Mon Sep 17 00:00:00 2001 From: James Allen Date: Mon, 7 Dec 2015 11:49:55 +0000 Subject: [PATCH 054/491] Use configurable app name if present --- services/real-time/app.coffee | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/services/real-time/app.coffee b/services/real-time/app.coffee index dff0fd17bd..2799ab933a 100644 --- a/services/real-time/app.coffee +++ b/services/real-time/app.coffee @@ -11,7 +11,7 @@ logger = require "logger-sharelatex" logger.initialize("real-time-sharelatex") Metrics = require("metrics-sharelatex") -Metrics.initialize("real-time") +Metrics.initialize(Settings.appName or "real-time") Metrics.event_loop.monitor(logger) rclient = redis.createClient(Settings.redis.web) From 030abc5340cc4e99d38c6c26551d67af8066928d Mon Sep 17 00:00:00 2001 From: James Allen Date: Wed, 20 Jan 2016 17:51:24 +0000 Subject: [PATCH 055/491] Don't flush to track changes now that this happens in doc updater --- .../app/coffee/TrackChangesManager.coffee | 16 ------- .../app/coffee/WebsocketController.coffee | 4 -- .../real-time/config/settings.defaults.coffee | 2 - .../coffee/LeaveProjectTests.coffee | 16 +------ .../helpers/MockTrackChangesServer.coffee | 21 -------- .../coffee/TrackChangesManagerTests.coffee | 48 ------------------- .../coffee/WebsocketControllerTests.coffee | 11 ----- 7 files changed, 1 insertion(+), 117 deletions(-) delete mode 100644 services/real-time/app/coffee/TrackChangesManager.coffee delete mode 100644 services/real-time/test/acceptance/coffee/helpers/MockTrackChangesServer.coffee delete mode 100644 services/real-time/test/unit/coffee/TrackChangesManagerTests.coffee diff --git a/services/real-time/app/coffee/TrackChangesManager.coffee b/services/real-time/app/coffee/TrackChangesManager.coffee deleted file mode 100644 index 49fa956cf6..0000000000 --- a/services/real-time/app/coffee/TrackChangesManager.coffee +++ /dev/null @@ -1,16 +0,0 @@ -settings = require "settings-sharelatex" -request = require "request" -logger = require "logger-sharelatex" - -module.exports = TrackChangesManager = - flushProject: (project_id, callback = (error) ->) -> - logger.log project_id: project_id, "flushing project in track-changes api" - url = "#{settings.apis.trackchanges.url}/project/#{project_id}/flush" - request.post url, (error, res, body) -> - return callback(error) if error? - if 200 <= res.statusCode < 300 - callback(null) - else - error = new Error("track-changes api responded with non-success code: #{res.statusCode}") - logger.error err: error, project_id: project_id, "error flushing project in track-changes api" - callback(error) \ No newline at end of file diff --git a/services/real-time/app/coffee/WebsocketController.coffee b/services/real-time/app/coffee/WebsocketController.coffee index d578c36622..bb2049f61d 100644 --- a/services/real-time/app/coffee/WebsocketController.coffee +++ b/services/real-time/app/coffee/WebsocketController.coffee @@ -4,7 +4,6 @@ WebApiManager = require "./WebApiManager" AuthorizationManager = require "./AuthorizationManager" DocumentUpdaterManager = require "./DocumentUpdaterManager" ConnectedUsersManager = require "./ConnectedUsersManager" -TrackChangesManager = require "./TrackChangesManager" WebsocketLoadBalancer = require "./WebsocketLoadBalancer" Utils = require "./Utils" @@ -68,9 +67,6 @@ module.exports = WebsocketController = DocumentUpdaterManager.flushProjectToMongoAndDelete project_id, (err) -> if err? logger.error {err, project_id, user_id, client_id: client.id}, "error flushing to doc updater after leaving project" - TrackChangesManager.flushProject project_id, (err) -> - if err? - logger.error {err, project_id, user_id, client_id: client.id}, "error flushing to track changes after leaving project" callback() , WebsocketController.FLUSH_IF_EMPTY_DELAY diff --git a/services/real-time/config/settings.defaults.coffee b/services/real-time/config/settings.defaults.coffee index eaf5e7d61e..5e6a3691ab 100644 --- a/services/real-time/config/settings.defaults.coffee +++ b/services/real-time/config/settings.defaults.coffee @@ -19,8 +19,6 @@ module.exports = pass: "password" documentupdater: url: "http://localhost:3003" - trackchanges: - url: "http://localhost:3015" security: sessionSecret: "secret-please-change" diff --git a/services/real-time/test/acceptance/coffee/LeaveProjectTests.coffee b/services/real-time/test/acceptance/coffee/LeaveProjectTests.coffee index 2d1361a18b..2d8c8c584f 100644 --- a/services/real-time/test/acceptance/coffee/LeaveProjectTests.coffee +++ b/services/real-time/test/acceptance/coffee/LeaveProjectTests.coffee @@ -1,15 +1,12 @@ RealTimeClient = require "./helpers/RealTimeClient" MockDocUpdaterServer = require "./helpers/MockDocUpdaterServer" -MockTrackChangesServer = require "./helpers/MockTrackChangesServer" FixturesManager = require "./helpers/FixturesManager" async = require "async" describe "leaveProject", -> before (done) -> - MockDocUpdaterServer.run (error) -> - return done(error) if error? - MockTrackChangesServer.run done + MockDocUpdaterServer.run done describe "with other clients in the project", -> before (done) -> @@ -67,11 +64,6 @@ describe "leaveProject", -> MockDocUpdaterServer.deleteProject .calledWith(@project_id) .should.equal false - - it "should not flush the project in track changes", -> - MockTrackChangesServer.flushProject - .calledWith(@project_id) - .should.equal false describe "with no other clients in the project", -> before (done) -> @@ -106,9 +98,3 @@ describe "leaveProject", -> MockDocUpdaterServer.deleteProject .calledWith(@project_id) .should.equal true - - it "should flush the project in track changes", -> - MockTrackChangesServer.flushProject - .calledWith(@project_id) - .should.equal true - \ No newline at end of file diff --git a/services/real-time/test/acceptance/coffee/helpers/MockTrackChangesServer.coffee b/services/real-time/test/acceptance/coffee/helpers/MockTrackChangesServer.coffee deleted file mode 100644 index e7ddbf5cea..0000000000 --- a/services/real-time/test/acceptance/coffee/helpers/MockTrackChangesServer.coffee +++ /dev/null @@ -1,21 +0,0 @@ -sinon = require "sinon" -express = require "express" - -module.exports = MockTrackChangesServer = - flushProject: sinon.stub().callsArg(1) - - flushProjectRequest: (req, res, next) -> - {project_id} = req.params - MockTrackChangesServer.flushProject project_id, (error) -> - return next(error) if error? - res.sendStatus 204 - - running: false - run: (callback = (error) ->) -> - if MockTrackChangesServer.running - return callback() - app = express() - app.post "/project/:project_id/flush", MockTrackChangesServer.flushProjectRequest - app.listen 3015, (error) -> - MockTrackChangesServer.running = true - callback(error) \ No newline at end of file diff --git a/services/real-time/test/unit/coffee/TrackChangesManagerTests.coffee b/services/real-time/test/unit/coffee/TrackChangesManagerTests.coffee deleted file mode 100644 index 69c11c87b1..0000000000 --- a/services/real-time/test/unit/coffee/TrackChangesManagerTests.coffee +++ /dev/null @@ -1,48 +0,0 @@ -chai = require('chai') -chai.should() -sinon = require("sinon") -modulePath = "../../../app/js/TrackChangesManager" -SandboxedModule = require('sandboxed-module') - -describe "TrackChangesManager", -> - beforeEach -> - @TrackChangesManager = SandboxedModule.require modulePath, requires: - "request" : @request = sinon.stub() - "settings-sharelatex": @settings = - apis: - trackchanges: - url: "trackchanges.sharelatex.com" - "logger-sharelatex": @logger = {log: sinon.stub(), error: sinon.stub()} - @project_id = "project-id-123" - @callback = sinon.stub() - - describe "flushProject", -> - describe "with a successful response code", -> - beforeEach -> - @request.post = sinon.stub().callsArgWith(1, null, statusCode: 204, "") - @TrackChangesManager.flushProject @project_id, @callback - - it "should flush the project in the track changes api", -> - @request.post - .calledWith("#{@settings.apis.trackchanges.url}/project/#{@project_id}/flush") - .should.equal true - - it "should call the callback without an error", -> - @callback.calledWith(null).should.equal true - - describe "with a failed response code", -> - beforeEach -> - @request.post = sinon.stub().callsArgWith(1, null, statusCode: 500, "") - @TrackChangesManager.flushProject @project_id, @callback - - it "should call the callback with an error", -> - @callback.calledWith(new Error("track-changes api responded with a non-success code: 500")).should.equal true - - it "should log the error", -> - @logger.error - .calledWith({ - err: new Error("track-changes api responded with a non-success code: 500") - project_id: @project_id - }, "error flushing project in track-changes api") - .should.equal true - diff --git a/services/real-time/test/unit/coffee/WebsocketControllerTests.coffee b/services/real-time/test/unit/coffee/WebsocketControllerTests.coffee index 2f9e66d1ef..d648b39c8c 100644 --- a/services/real-time/test/unit/coffee/WebsocketControllerTests.coffee +++ b/services/real-time/test/unit/coffee/WebsocketControllerTests.coffee @@ -29,7 +29,6 @@ describe 'WebsocketController', -> "./WebApiManager": @WebApiManager = {} "./AuthorizationManager": @AuthorizationManager = {} "./DocumentUpdaterManager": @DocumentUpdaterManager = {} - "./TrackChangesManager": @TrackChangesManager = {} "./ConnectedUsersManager": @ConnectedUsersManager = {} "./WebsocketLoadBalancer": @WebsocketLoadBalancer = {} "logger-sharelatex": @logger = { log: sinon.stub(), error: sinon.stub() } @@ -120,7 +119,6 @@ describe 'WebsocketController', -> describe "leaveProject", -> beforeEach -> @DocumentUpdaterManager.flushProjectToMongoAndDelete = sinon.stub().callsArg(1) - @TrackChangesManager.flushProject = sinon.stub().callsArg(1) @ConnectedUsersManager.markUserAsDisconnected = sinon.stub().callsArg(2) @WebsocketLoadBalancer.emitToRoom = sinon.stub() @clientsInRoom = [] @@ -154,11 +152,6 @@ describe 'WebsocketController', -> .calledWith(@project_id) .should.equal true - it "should flush the changes in the track changes api", -> - @TrackChangesManager.flushProject - .calledWith(@project_id) - .should.equal true - it "should increment the leave-project metric", -> @metrics.inc.calledWith("editor.leave-project").should.equal true @@ -170,10 +163,6 @@ describe 'WebsocketController', -> it "should not flush the project in the document updater", -> @DocumentUpdaterManager.flushProjectToMongoAndDelete .called.should.equal false - - it "should not flush the changes in the track changes api", -> - @TrackChangesManager.flushProject - .called.should.equal false describe "joinDoc", -> beforeEach -> From 343ec9d708cc8b386d159977be8a3885a7202776 Mon Sep 17 00:00:00 2001 From: James Allen Date: Thu, 26 May 2016 15:46:45 +0100 Subject: [PATCH 056/491] Add in flags that track how often each callback is called --- services/real-time/app/coffee/Router.coffee | 2 +- services/real-time/app/coffee/WebsocketController.coffee | 6 +++++- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/services/real-time/app/coffee/Router.coffee b/services/real-time/app/coffee/Router.coffee index 0fee1cb837..9a055e0b86 100644 --- a/services/real-time/app/coffee/Router.coffee +++ b/services/real-time/app/coffee/Router.coffee @@ -23,7 +23,7 @@ module.exports = Router = attrs.err = error logger.error attrs, "server side error in #{method}" # Don't return raw error to prevent leaking server side info - return callback {message: "Something went wrong"} + return callback {message: "Something went wrong in real-time service"} configure: (app, io, session) -> app.set("io", io) diff --git a/services/real-time/app/coffee/WebsocketController.coffee b/services/real-time/app/coffee/WebsocketController.coffee index bb2049f61d..af7b6aac98 100644 --- a/services/real-time/app/coffee/WebsocketController.coffee +++ b/services/real-time/app/coffee/WebsocketController.coffee @@ -149,12 +149,16 @@ module.exports = WebsocketController = applyOtUpdate: (client, doc_id, update, callback = (error) ->) -> + cbc_0 = 0 # Callback counter + cbc_1 = 0 Utils.getClientAttributes client, ["user_id", "project_id"], (error, {user_id, project_id}) -> + cbc_0++ return callback(error) if error? return callback(new Error("no project_id found on client")) if !project_id? # Omit this logging for now since it's likely too noisey #logger.log {user_id, project_id, doc_id, client_id: client.id, update: update}, "applying update" AuthorizationManager.assertClientCanEditProject client, (error) -> + cbc_1++ if error? logger.error {err: error, doc_id, client_id: client.id, version: update.v}, "client is not authorized to make update" setTimeout () -> @@ -169,7 +173,7 @@ module.exports = WebsocketController = metrics.set "editor.active-projects", project_id, 0.3 metrics.set "editor.active-users", user_id, 0.3 - logger.log {user_id, doc_id, project_id, client_id: client.id, version: update.v}, "sending update to doc updater" + logger.log {user_id, doc_id, project_id, client_id: client.id, version: update.v, cbc_0, cbc_1}, "sending update to doc updater" DocumentUpdaterManager.queueChange project_id, doc_id, update, (error) -> if error? From f4a465ea693695b1d89d1b931124e85388211c42 Mon Sep 17 00:00:00 2001 From: James Allen Date: Tue, 31 May 2016 11:49:51 +0100 Subject: [PATCH 057/491] Return a 'not authorized' error if the user is not logged in/authorized --- services/real-time/app/coffee/Router.coffee | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/services/real-time/app/coffee/Router.coffee b/services/real-time/app/coffee/Router.coffee index 9a055e0b86..c0e7313c39 100644 --- a/services/real-time/app/coffee/Router.coffee +++ b/services/real-time/app/coffee/Router.coffee @@ -21,9 +21,13 @@ module.exports = Router = attrs[key] = value attrs.client_id = client.id attrs.err = error - logger.error attrs, "server side error in #{method}" - # Don't return raw error to prevent leaking server side info - return callback {message: "Something went wrong in real-time service"} + if error.message == "not authorized" + logger.warn attrs, "client is not authorized" + return callback {message: error.message} + else + logger.error attrs, "server side error in #{method}" + # Don't return raw error to prevent leaking server side info + return callback {message: "Something went wrong in real-time service"} configure: (app, io, session) -> app.set("io", io) @@ -99,4 +103,4 @@ module.exports = Router = if err? Router._handleError callback, err, client, "applyOtUpdate", {doc_id, update} else - callback() \ No newline at end of file + callback() From 51939512acd97229bf32f8d37642781002962735 Mon Sep 17 00:00:00 2001 From: James Allen Date: Tue, 31 May 2016 14:21:23 +0100 Subject: [PATCH 058/491] Return semantic error if doc ops range is not loaded --- .../app/coffee/DocumentUpdaterManager.coffee | 5 +++++ services/real-time/app/coffee/Router.coffee | 4 ++-- .../unit/coffee/DocumentUpdaterManagerTests.coffee | 14 +++++++++++++- 3 files changed, 20 insertions(+), 3 deletions(-) diff --git a/services/real-time/app/coffee/DocumentUpdaterManager.coffee b/services/real-time/app/coffee/DocumentUpdaterManager.coffee index a2c9fe54f8..9e25585b1c 100644 --- a/services/real-time/app/coffee/DocumentUpdaterManager.coffee +++ b/services/real-time/app/coffee/DocumentUpdaterManager.coffee @@ -23,6 +23,11 @@ module.exports = DocumentUpdaterManager = catch error return callback(error) callback null, body?.lines, body?.version, body?.ops + else if res.statusCode == 422 # Unprocessable Entity + err = new Error("doc updater could not load requested ops") + err.statusCode = res.statusCode + logger.warn {err, project_id, doc_id, url, fromVersion}, "doc updater could not load requested ops" + callback err else err = new Error("doc updater returned a non-success status code: #{res.statusCode}") err.statusCode = res.statusCode diff --git a/services/real-time/app/coffee/Router.coffee b/services/real-time/app/coffee/Router.coffee index c0e7313c39..a8bdd7c7b1 100644 --- a/services/real-time/app/coffee/Router.coffee +++ b/services/real-time/app/coffee/Router.coffee @@ -21,8 +21,8 @@ module.exports = Router = attrs[key] = value attrs.client_id = client.id attrs.err = error - if error.message == "not authorized" - logger.warn attrs, "client is not authorized" + if error.message in ["not authorized", "doc updater could not load requested ops"] + logger.warn attrs, error.message return callback {message: error.message} else logger.error attrs, "server side error in #{method}" diff --git a/services/real-time/test/unit/coffee/DocumentUpdaterManagerTests.coffee b/services/real-time/test/unit/coffee/DocumentUpdaterManagerTests.coffee index 8a9d335b1a..8e75da1d0c 100644 --- a/services/real-time/test/unit/coffee/DocumentUpdaterManagerTests.coffee +++ b/services/real-time/test/unit/coffee/DocumentUpdaterManagerTests.coffee @@ -17,7 +17,7 @@ describe 'DocumentUpdaterManager', -> @DocumentUpdaterManager = SandboxedModule.require modulePath, requires: 'settings-sharelatex':@settings - 'logger-sharelatex': @logger = {log: sinon.stub(), error: sinon.stub()} + 'logger-sharelatex': @logger = {log: sinon.stub(), error: sinon.stub(), warn: sinon.stub()} 'request': @request = {} 'redis-sharelatex' : createClient: () => @rclient @@ -50,6 +50,18 @@ describe 'DocumentUpdaterManager', -> it "should return an error to the callback", -> @callback.calledWith(@error).should.equal true + describe "when the document updater returns a 422 status code", -> + beforeEach -> + @request.get = sinon.stub().callsArgWith(1, null, { statusCode: 422 }, "") + @DocumentUpdaterManager.getDocument @project_id, @doc_id, @fromVersion, @callback + + it "should return the callback with an error", -> + err = new Error("doc updater could not load requested ops") + err.statusCode = 422 + @callback + .calledWith(err) + .should.equal true + describe "when the document updater returns a failure error code", -> beforeEach -> @request.get = sinon.stub().callsArgWith(1, null, { statusCode: 500 }, "") From 9ab19c5d0309efac937f21fadb1f5e91dc1fd556 Mon Sep 17 00:00:00 2001 From: Brian Gough Date: Fri, 2 Sep 2016 16:34:14 +0100 Subject: [PATCH 059/491] avoid double callback --- services/real-time/app/coffee/WebsocketController.coffee | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/services/real-time/app/coffee/WebsocketController.coffee b/services/real-time/app/coffee/WebsocketController.coffee index af7b6aac98..4e537a664e 100644 --- a/services/real-time/app/coffee/WebsocketController.coffee +++ b/services/real-time/app/coffee/WebsocketController.coffee @@ -113,7 +113,7 @@ module.exports = WebsocketController = AuthorizationManager.assertClientCanViewProject client, (error) -> if error? logger.warn {client_id: client.id, project_id, user_id}, "silently ignoring unauthorized updateClientPosition. Client likely hasn't called joinProject yet." - callback() + return callback() cursorData.id = client.id cursorData.user_id = user_id if user_id? cursorData.email = email if email? From ef85bce3b8bbe63efacabbcd36ec5d1944b77b3a Mon Sep 17 00:00:00 2001 From: Brian Gough Date: Fri, 2 Sep 2016 16:35:00 +0100 Subject: [PATCH 060/491] track permissions when clients join and leave docs --- .../app/coffee/AuthorizationManager.coffee | 26 +++- .../app/coffee/WebsocketController.coffee | 12 +- .../coffee/AuthorizationManagerTests.coffee | 130 +++++++++++++++++- .../coffee/WebsocketControllerTests.coffee | 14 +- 4 files changed, 169 insertions(+), 13 deletions(-) diff --git a/services/real-time/app/coffee/AuthorizationManager.coffee b/services/real-time/app/coffee/AuthorizationManager.coffee index 273e9697cf..b4cf854238 100644 --- a/services/real-time/app/coffee/AuthorizationManager.coffee +++ b/services/real-time/app/coffee/AuthorizationManager.coffee @@ -12,4 +12,28 @@ module.exports = AuthorizationManager = if allowed callback null else - callback new Error("not authorized") \ No newline at end of file + callback new Error("not authorized") + + assertClientCanViewProjectAndDoc: (client, doc_id, callback = (error) ->) -> + AuthorizationManager.assertClientCanViewProject client, (error) -> + return callback(error) if error? + AuthorizationManager._assertClientCanAccessDoc client, doc_id, callback + + assertClientCanEditProjectAndDoc: (client, doc_id, callback = (error) ->) -> + AuthorizationManager.assertClientCanEditProject client, (error) -> + return callback(error) if error? + AuthorizationManager._assertClientCanAccessDoc client, doc_id, callback + + _assertClientCanAccessDoc: (client, doc_id, callback = (error) ->) -> + client.get "doc:#{doc_id}", (error, status) -> + return callback(error) if error? + if status? and status is "allowed" + callback null + else + callback new Error("not authorized") + + addAccessToDoc: (client, doc_id, callback = (error) ->) -> + client.set("doc:#{doc_id}", "allowed", callback) + + removeAccessToDoc: (client, doc_id, callback = (error) ->) -> + client.del("doc:#{doc_id}", callback) diff --git a/services/real-time/app/coffee/WebsocketController.coffee b/services/real-time/app/coffee/WebsocketController.coffee index 4e537a664e..0f04981d05 100644 --- a/services/real-time/app/coffee/WebsocketController.coffee +++ b/services/real-time/app/coffee/WebsocketController.coffee @@ -91,6 +91,7 @@ module.exports = WebsocketController = logger.err {err, project_id, doc_id, fromVersion, line, client_id: client.id}, "error encoding line uri component" return callback(err) escapedLines.push line + AuthorizationManager.addAccessToDoc client, doc_id client.join(doc_id) callback null, escapedLines, version, ops logger.log {user_id, project_id, doc_id, fromVersion, client_id: client.id}, "client joined doc" @@ -99,8 +100,9 @@ module.exports = WebsocketController = metrics.inc "editor.leave-doc" Utils.getClientAttributes client, ["project_id", "user_id"], (error, {project_id, user_id}) -> logger.log {user_id, project_id, doc_id, client_id: client.id}, "client leaving doc" - client.leave doc_id - callback() + client.leave doc_id + AuthorizationManager.removeAccessToDoc client, doc_id # may not be needed, could block updates? + callback() updateClientPosition: (client, cursorData, callback = (error) ->) -> metrics.inc "editor.update-client-position", 0.1 @@ -110,7 +112,7 @@ module.exports = WebsocketController = return callback(error) if error? logger.log {user_id, project_id, client_id: client.id, cursorData: cursorData}, "updating client position" - AuthorizationManager.assertClientCanViewProject client, (error) -> + AuthorizationManager.assertClientCanViewProjectAndDoc client, cursorData.doc_id, (error) -> if error? logger.warn {client_id: client.id, project_id, user_id}, "silently ignoring unauthorized updateClientPosition. Client likely hasn't called joinProject yet." return callback() @@ -133,7 +135,7 @@ module.exports = WebsocketController = cursorData.name = "Anonymous" callback() WebsocketLoadBalancer.emitToRoom(project_id, "clientTracking.clientUpdated", cursorData) - + getConnectedUsers: (client, callback = (error, users) ->) -> metrics.inc "editor.get-connected-users" Utils.getClientAttributes client, ["project_id", "user_id"], (error, {project_id, user_id}) -> @@ -157,7 +159,7 @@ module.exports = WebsocketController = return callback(new Error("no project_id found on client")) if !project_id? # Omit this logging for now since it's likely too noisey #logger.log {user_id, project_id, doc_id, client_id: client.id, update: update}, "applying update" - AuthorizationManager.assertClientCanEditProject client, (error) -> + AuthorizationManager.assertClientCanEditProjectAndDoc client, doc_id, (error) -> cbc_1++ if error? logger.error {err: error, doc_id, client_id: client.id, version: update.v}, "client is not authorized to make update" diff --git a/services/real-time/test/unit/coffee/AuthorizationManagerTests.coffee b/services/real-time/test/unit/coffee/AuthorizationManagerTests.coffee index a4741ab4d8..9856684247 100644 --- a/services/real-time/test/unit/coffee/AuthorizationManagerTests.coffee +++ b/services/real-time/test/unit/coffee/AuthorizationManagerTests.coffee @@ -10,7 +10,15 @@ describe 'AuthorizationManager', -> beforeEach -> @client = params: {} - get: (param, cb) -> cb null, @params[param] + get: (param, cb) -> + cb null, @params[param] + set: (param, value, cb) -> + @params[param] = value + cb() + del: (param, cb) -> + delete @params[param] + cb() + @AuthorizationManager = SandboxedModule.require modulePath, requires: {} describe "assertClientCanViewProject", -> @@ -61,4 +69,122 @@ describe 'AuthorizationManager', -> @client.params.privilege_level = "unknown" @AuthorizationManager.assertClientCanEditProject @client, (error) -> error.message.should.equal "not authorized" - done() \ No newline at end of file + done() + + # check doc access for project + + describe "assertClientCanViewProjectAndDoc", -> + beforeEach () -> + @doc_id = "12345" + @callback = sinon.stub() + @client.params = {} + + describe "when not authorised at the project level", -> + beforeEach () -> + @client.params.privilege_level = "unknown" + + it "should not allow access", () -> + @AuthorizationManager.assertClientCanViewProjectAndDoc @client, @doc_id, @callback + @callback + .calledWith(new Error("not authorised")) + .should.equal true + + describe "even when authorised at the doc level", -> + beforeEach (done) -> + @AuthorizationManager.addAccessToDoc @client, @doc_id, done + + it "should not allow access", () -> + @AuthorizationManager.assertClientCanViewProjectAndDoc @client, @doc_id, @callback + @callback + .calledWith(new Error("not authorised")) + .should.equal true + + describe "when authorised at the project level", -> + beforeEach () -> + @client.params.privilege_level = "readOnly" + + describe "and not authorised at the document level", -> + it "should not allow access", () -> + @AuthorizationManager.assertClientCanViewProjectAndDoc @client, @doc_id, @callback + @callback + .calledWith(new Error("not authorised")) + .should.equal true + + describe "and authorised at the document level", -> + beforeEach (done) -> + @AuthorizationManager.addAccessToDoc @client, @doc_id, done + + it "should allow access", () -> + @AuthorizationManager.assertClientCanViewProjectAndDoc @client, @doc_id, @callback + @callback + .calledWith(null) + .should.equal true + + describe "when document authorisation is added and then removed", -> + beforeEach (done) -> + @AuthorizationManager.addAccessToDoc @client, @doc_id, () => + @AuthorizationManager.removeAccessToDoc @client, @doc_id, done + + it "should deny access", () -> + @AuthorizationManager.assertClientCanViewProjectAndDoc @client, @doc_id, @callback + @callback + .calledWith(new Error("not authorised")) + .should.equal true + + describe "assertClientCanEditProjectAndDoc", -> + beforeEach () -> + @doc_id = "12345" + @callback = sinon.stub() + @client.params = {} + + describe "when not authorised at the project level", -> + beforeEach () -> + @client.params.privilege_level = "readOnly" + + it "should not allow access", () -> + @AuthorizationManager.assertClientCanEditProjectAndDoc @client, @doc_id, @callback + @callback + .calledWith(new Error("not authorised")) + .should.equal true + + describe "even when authorised at the doc level", -> + beforeEach (done) -> + @AuthorizationManager.addAccessToDoc @client, @doc_id, done + + it "should not allow access", () -> + @AuthorizationManager.assertClientCanEditProjectAndDoc @client, @doc_id, @callback + @callback + .calledWith(new Error("not authorised")) + .should.equal true + + describe "when authorised at the project level", -> + beforeEach () -> + @client.params.privilege_level = "readAndWrite" + + describe "and not authorised at the document level", -> + it "should not allow access", () -> + @AuthorizationManager.assertClientCanEditProjectAndDoc @client, @doc_id, @callback + @callback + .calledWith(new Error("not authorised")) + .should.equal true + + describe "and authorised at the document level", -> + beforeEach (done) -> + @AuthorizationManager.addAccessToDoc @client, @doc_id, done + + it "should allow access", () -> + @AuthorizationManager.assertClientCanEditProjectAndDoc @client, @doc_id, @callback + @callback + .calledWith(null) + .should.equal true + + describe "when document authorisation is added and then removed", -> + beforeEach (done) -> + @AuthorizationManager.addAccessToDoc @client, @doc_id, () => + @AuthorizationManager.removeAccessToDoc @client, @doc_id, done + + it "should deny access", () -> + @AuthorizationManager.assertClientCanEditProjectAndDoc @client, @doc_id, @callback + @callback + .calledWith(new Error("not authorised")) + .should.equal true diff --git a/services/real-time/test/unit/coffee/WebsocketControllerTests.coffee b/services/real-time/test/unit/coffee/WebsocketControllerTests.coffee index d648b39c8c..48eccd4bee 100644 --- a/services/real-time/test/unit/coffee/WebsocketControllerTests.coffee +++ b/services/real-time/test/unit/coffee/WebsocketControllerTests.coffee @@ -172,7 +172,7 @@ describe 'WebsocketController', -> @ops = ["mock", "ops"] @client.params.project_id = @project_id - + @AuthorizationManager.addAccessToDoc = sinon.stub() @AuthorizationManager.assertClientCanViewProject = sinon.stub().callsArgWith(1, null) @DocumentUpdaterManager.getDocument = sinon.stub().callsArgWith(3, null, @doc_lines, @version, @ops) @@ -190,6 +190,11 @@ describe 'WebsocketController', -> @DocumentUpdaterManager.getDocument .calledWith(@project_id, @doc_id, @fromVersion) .should.equal true + + it "should add permissions for the client to access the doc", -> + @AuthorizationManager.addAccessToDoc + .calledWith(@client, @doc_id) + .should.equal true it "should join the client to room for the doc_id", -> @client.join @@ -287,7 +292,7 @@ describe 'WebsocketController', -> beforeEach -> @WebsocketLoadBalancer.emitToRoom = sinon.stub() @ConnectedUsersManager.updateUserPosition = sinon.stub().callsArgWith(4) - @AuthorizationManager.assertClientCanViewProject = sinon.stub().callsArgWith(1, null) + @AuthorizationManager.assertClientCanViewProjectAndDoc = sinon.stub().callsArgWith(2, null) @update = { doc_id: @doc_id = "doc-id-123" row: @row = 42 @@ -362,7 +367,7 @@ describe 'WebsocketController', -> @update = {op: {p: 12, t: "foo"}} @client.params.user_id = @user_id @client.params.project_id = @project_id - @AuthorizationManager.assertClientCanEditProject = sinon.stub().callsArg(1) + @AuthorizationManager.assertClientCanEditProjectAndDoc = sinon.stub().callsArg(2) @DocumentUpdaterManager.queueChange = sinon.stub().callsArg(3) describe "succesfully", -> @@ -410,7 +415,7 @@ describe 'WebsocketController', -> describe "when not authorized", -> beforeEach -> @client.disconnect = sinon.stub() - @AuthorizationManager.assertClientCanEditProject = sinon.stub().callsArgWith(1, @error = new Error("not authorized")) + @AuthorizationManager.assertClientCanEditProjectAndDoc = sinon.stub().callsArgWith(2, @error = new Error("not authorized")) @WebsocketController.applyOtUpdate @client, @doc_id, @update, @callback # This happens in a setTimeout to allow the client a chance to receive the error first. @@ -423,4 +428,3 @@ describe 'WebsocketController', -> it "should call the callback with the error", -> @callback.calledWith(@error).should.equal true - \ No newline at end of file From 8ffec68250aa6fc0e0105f89ef95c5e047df94a7 Mon Sep 17 00:00:00 2001 From: Brian Gough Date: Mon, 5 Sep 2016 12:35:49 +0100 Subject: [PATCH 061/491] add comment about fallback case --- services/real-time/app/coffee/WebsocketController.coffee | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/services/real-time/app/coffee/WebsocketController.coffee b/services/real-time/app/coffee/WebsocketController.coffee index 0f04981d05..d2aef5b424 100644 --- a/services/real-time/app/coffee/WebsocketController.coffee +++ b/services/real-time/app/coffee/WebsocketController.coffee @@ -101,7 +101,10 @@ module.exports = WebsocketController = Utils.getClientAttributes client, ["project_id", "user_id"], (error, {project_id, user_id}) -> logger.log {user_id, project_id, doc_id, client_id: client.id}, "client leaving doc" client.leave doc_id - AuthorizationManager.removeAccessToDoc client, doc_id # may not be needed, could block updates? + # we could remove permission when user leaves a doc, but because + # the connection is per-project, we continue to allow access + # after the initial joinDoc since we know they are already authorised. + ## AuthorizationManager.removeAccessToDoc client, doc_id callback() updateClientPosition: (client, cursorData, callback = (error) ->) -> From 185bc7e6353ea3ad3d68755a4cdbe25aa844264b Mon Sep 17 00:00:00 2001 From: Shane Kilkelly Date: Wed, 7 Sep 2016 08:58:35 +0100 Subject: [PATCH 062/491] Update session code --- services/real-time/app/coffee/Router.coffee | 34 +++++++++++---------- 1 file changed, 18 insertions(+), 16 deletions(-) diff --git a/services/real-time/app/coffee/Router.coffee b/services/real-time/app/coffee/Router.coffee index a8bdd7c7b1..64fdb6931d 100644 --- a/services/real-time/app/coffee/Router.coffee +++ b/services/real-time/app/coffee/Router.coffee @@ -35,69 +35,71 @@ module.exports = Router = app.get "/clients/:client_id", HttpController.getConnectedClient app.post "/project/:project_id/message/:message", httpAuth, bodyParser.json(limit: "5mb"), HttpApiController.sendMessage - + session.on 'connection', (error, client, session) -> if error? logger.err err: error, "error when client connected" client?.disconnect() return - + metrics.inc('socket-io.connection') - + logger.log session: session, client_id: client.id, "client connected" - - if !session or !session.user? - user = {_id: "anonymous-user"} - else + + if session?.passport?.user? + user = session.passport.user + else if session?.user? user = session.user - + else + user = {_id: "anonymous-user"} + client.on "joinProject", (data = {}, callback) -> WebsocketController.joinProject client, user, data.project_id, (err, args...) -> if err? Router._handleError callback, err, client, "joinProject", {project_id: data.project_id, user_id: user?.id} else callback(null, args...) - + client.on "disconnect", () -> metrics.inc('socket-io.disconnect') WebsocketController.leaveProject io, client, (err) -> if err? Router._handleError null, err, client, "leaveProject" - - + + client.on "joinDoc", (doc_id, fromVersion, callback) -> # fromVersion is optional if typeof fromVersion == "function" callback = fromVersion fromVersion = -1 - + WebsocketController.joinDoc client, doc_id, fromVersion, (err, args...) -> if err? Router._handleError callback, err, client, "joinDoc", {doc_id, fromVersion} else callback(null, args...) - + client.on "leaveDoc", (doc_id, callback) -> WebsocketController.leaveDoc client, doc_id, (err, args...) -> if err? Router._handleError callback, err, client, "leaveDoc" else callback(null, args...) - + client.on "clientTracking.getConnectedUsers", (callback = (error, users) ->) -> WebsocketController.getConnectedUsers client, (err, users) -> if err? Router._handleError callback, err, client, "clientTracking.getConnectedUsers" else callback(null, users) - + client.on "clientTracking.updatePosition", (cursorData, callback = (error) ->) -> WebsocketController.updateClientPosition client, cursorData, (err) -> if err? Router._handleError callback, err, client, "clientTracking.updatePosition" else callback() - + client.on "applyOtUpdate", (doc_id, update, callback = (error) ->) -> WebsocketController.applyOtUpdate client, doc_id, update, (err) -> if err? From 59d042e2643818887c279c7588c91432ed920875 Mon Sep 17 00:00:00 2001 From: James Allen Date: Mon, 24 Oct 2016 16:36:09 +0100 Subject: [PATCH 063/491] Add end point to start draining clients --- .../real-time/app/coffee/DrainManager.coffee | 26 ++++++++ .../app/coffee/HttpApiController.coffee | 11 +++- services/real-time/app/coffee/Router.coffee | 2 + .../test/unit/coffee/DrainManagerTests.coffee | 64 +++++++++++++++++++ 4 files changed, 102 insertions(+), 1 deletion(-) create mode 100644 services/real-time/app/coffee/DrainManager.coffee create mode 100644 services/real-time/test/unit/coffee/DrainManagerTests.coffee diff --git a/services/real-time/app/coffee/DrainManager.coffee b/services/real-time/app/coffee/DrainManager.coffee new file mode 100644 index 0000000000..08caac3c28 --- /dev/null +++ b/services/real-time/app/coffee/DrainManager.coffee @@ -0,0 +1,26 @@ +logger = require "logger-sharelatex" + +module.exports = + startDrain: (io, rate) -> + # Clear out any old interval + clearInterval @interval + if rate == 0 + return + @interval = setInterval () => + @reconnectNClients(io, rate) + , 1000 + + RECONNECTED_CLIENTS: {} + reconnectNClients: (io, N) -> + drainedCount = 0 + for client in io.sockets.clients() + logger.log {client_id: client.id, already_reconnecting: @RECONNECTED_CLIENTS[client.id]}, "Considering client" + if !@RECONNECTED_CLIENTS[client.id] + @RECONNECTED_CLIENTS[client.id] = true + logger.log {client_id: client.id}, "Asking client to reconnect gracefully" + client.emit "reconnectGracefully" + drainedCount++ + if drainedCount == N + break + if drainedCount < N + logger.log "All clients have been told to reconnectGracefully" \ No newline at end of file diff --git a/services/real-time/app/coffee/HttpApiController.coffee b/services/real-time/app/coffee/HttpApiController.coffee index 1082a11b34..a2a9d4d23c 100644 --- a/services/real-time/app/coffee/HttpApiController.coffee +++ b/services/real-time/app/coffee/HttpApiController.coffee @@ -1,4 +1,5 @@ WebsocketLoadBalancer = require "./WebsocketLoadBalancer" +DrainManager = require "./DrainManager" logger = require "logger-sharelatex" module.exports = HttpApiController = @@ -9,4 +10,12 @@ module.exports = HttpApiController = WebsocketLoadBalancer.emitToRoom req.params.project_id, req.params.message, payload else WebsocketLoadBalancer.emitToRoom req.params.project_id, req.params.message, req.body - res.send 204 # No content \ No newline at end of file + res.send 204 # No content + + startDrain: (req, res, next) -> + io = req.app.get("io") + rate = req.query.rate or "4" + rate = parseInt(rate, 10) + logger.log {rate}, "setting client drain rate" + DrainManager.startDrain io, rate + res.send 204 \ No newline at end of file diff --git a/services/real-time/app/coffee/Router.coffee b/services/real-time/app/coffee/Router.coffee index 64fdb6931d..2f56020578 100644 --- a/services/real-time/app/coffee/Router.coffee +++ b/services/real-time/app/coffee/Router.coffee @@ -35,6 +35,8 @@ module.exports = Router = app.get "/clients/:client_id", HttpController.getConnectedClient app.post "/project/:project_id/message/:message", httpAuth, bodyParser.json(limit: "5mb"), HttpApiController.sendMessage + + app.post "/drain", httpAuth, HttpApiController.startDrain session.on 'connection', (error, client, session) -> if error? diff --git a/services/real-time/test/unit/coffee/DrainManagerTests.coffee b/services/real-time/test/unit/coffee/DrainManagerTests.coffee new file mode 100644 index 0000000000..b3cdaf1ca0 --- /dev/null +++ b/services/real-time/test/unit/coffee/DrainManagerTests.coffee @@ -0,0 +1,64 @@ +should = require('chai').should() +sinon = require "sinon" +SandboxedModule = require('sandboxed-module') +path = require "path" +modulePath = path.join __dirname, "../../../app/js/DrainManager" + +describe "DrainManager", -> + beforeEach -> + @DrainManager = SandboxedModule.require modulePath, requires: + "logger-sharelatex": @logger = log: sinon.stub() + @io = + sockets: + clients: sinon.stub() + + describe "reconnectNClients", -> + beforeEach -> + @clients = [] + for i in [0..9] + @clients[i] = { + id: i + emit: sinon.stub() + } + @io.sockets.clients.returns @clients + + describe "after first pass", -> + beforeEach -> + @DrainManager.reconnectNClients(@io, 3) + + it "should reconnect the first 3 clients", -> + for i in [0..2] + @clients[i].emit.calledWith("reconnectGracefully").should.equal true + + it "should not reconnect any more clients", -> + for i in [3..9] + @clients[i].emit.calledWith("reconnectGracefully").should.equal false + + describe "after second pass", -> + beforeEach -> + @DrainManager.reconnectNClients(@io, 3) + + it "should reconnect the next 3 clients", -> + for i in [3..5] + @clients[i].emit.calledWith("reconnectGracefully").should.equal true + + it "should not reconnect any more clients", -> + for i in [6..9] + @clients[i].emit.calledWith("reconnectGracefully").should.equal false + + it "should not reconnect the first 3 clients again", -> + for i in [0..2] + @clients[i].emit.calledOnce.should.equal true + + describe "after final pass", -> + beforeEach -> + @DrainManager.reconnectNClients(@io, 100) + + it "should not reconnect the first 6 clients again", -> + for i in [0..5] + @clients[i].emit.calledOnce.should.equal true + + it "should log out that it reached the end", -> + @logger.log + .calledWith("All clients have been told to reconnectGracefully") + .should.equal true From 7107d9adcdb7aa9480d50a027d775a44b95159cf Mon Sep 17 00:00:00 2001 From: James Allen Date: Mon, 24 Oct 2016 16:40:10 +0100 Subject: [PATCH 064/491] Delete logging that will be noisy in production --- services/real-time/app/coffee/DrainManager.coffee | 1 - 1 file changed, 1 deletion(-) diff --git a/services/real-time/app/coffee/DrainManager.coffee b/services/real-time/app/coffee/DrainManager.coffee index 08caac3c28..efdd636199 100644 --- a/services/real-time/app/coffee/DrainManager.coffee +++ b/services/real-time/app/coffee/DrainManager.coffee @@ -14,7 +14,6 @@ module.exports = reconnectNClients: (io, N) -> drainedCount = 0 for client in io.sockets.clients() - logger.log {client_id: client.id, already_reconnecting: @RECONNECTED_CLIENTS[client.id]}, "Considering client" if !@RECONNECTED_CLIENTS[client.id] @RECONNECTED_CLIENTS[client.id] = true logger.log {client_id: client.id}, "Asking client to reconnect gracefully" From 6fa2a81bafb760a2dfff9da3358881ade82f6e11 Mon Sep 17 00:00:00 2001 From: James Allen Date: Mon, 24 Oct 2016 16:54:56 +0100 Subject: [PATCH 065/491] Make breakout logic after draining N clients more clear --- services/real-time/app/coffee/DrainManager.coffee | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/services/real-time/app/coffee/DrainManager.coffee b/services/real-time/app/coffee/DrainManager.coffee index efdd636199..92b59b0751 100644 --- a/services/real-time/app/coffee/DrainManager.coffee +++ b/services/real-time/app/coffee/DrainManager.coffee @@ -19,7 +19,8 @@ module.exports = logger.log {client_id: client.id}, "Asking client to reconnect gracefully" client.emit "reconnectGracefully" drainedCount++ - if drainedCount == N + haveDrainedNClients = (drainedCount == N) + if haveDrainedNClients break if drainedCount < N logger.log "All clients have been told to reconnectGracefully" \ No newline at end of file From 2e0f5b74db8368c0ec945f7a897383f25280ba88 Mon Sep 17 00:00:00 2001 From: Brian Gough Date: Fri, 28 Oct 2016 15:40:03 +0100 Subject: [PATCH 066/491] send connectionAccepted/Rejected events on connect let the client know whether it has successfully authenticated --- services/real-time/app/coffee/Router.coffee | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/services/real-time/app/coffee/Router.coffee b/services/real-time/app/coffee/Router.coffee index 2f56020578..2cc655eafc 100644 --- a/services/real-time/app/coffee/Router.coffee +++ b/services/real-time/app/coffee/Router.coffee @@ -39,11 +39,22 @@ module.exports = Router = app.post "/drain", httpAuth, HttpApiController.startDrain session.on 'connection', (error, client, session) -> + if client? and error?.message?.match(/could not look up session by key/) + logger.err err: error, client: client?, session: session?, "invalid session" + # tell the client to reauthenticate if it has an invalid session key + client.emit("connectionRejected", {message: "invalid session"}) + client.disconnect() + return + if error? - logger.err err: error, "error when client connected" + logger.err err: error, client: client?, session: session?, "error when client connected" + client?.emit("connectionRejected", {message: "error"}) client?.disconnect() return + # send positive confirmation that the client has a valid connection + client.emit("connectionAccepted") + metrics.inc('socket-io.connection') logger.log session: session, client_id: client.id, "client connected" From 41868ddda3b92f6b85543b0d26dc763e6c8d6d57 Mon Sep 17 00:00:00 2001 From: Shane Kilkelly Date: Wed, 9 Nov 2016 12:06:32 +0000 Subject: [PATCH 067/491] Make real-time work with web sessions in redis-cluster --- services/real-time/app.coffee | 30 +++++++++++++++++++----------- services/real-time/package.json | 1 + 2 files changed, 20 insertions(+), 11 deletions(-) diff --git a/services/real-time/app.coffee b/services/real-time/app.coffee index 2799ab933a..d21b133022 100644 --- a/services/real-time/app.coffee +++ b/services/real-time/app.coffee @@ -1,20 +1,28 @@ +logger = require "logger-sharelatex" +logger.initialize("real-time-sharelatex") + express = require("express") session = require("express-session") redis = require("redis-sharelatex") +ioredis = require('ioredis') +Settings = require "settings-sharelatex" + +redisSessionsSettings = Settings.redis.websessions or Settings.redis.web + +if redisSessionsSettings?.cluster? + logger.log {}, "using redis cluster for web sessions" + rclient = new ioredis.Cluster(redisSessionsSettings.cluster) +else + rclient = redis.createClient(redisSessionsSettings) + RedisStore = require('connect-redis')(session) SessionSockets = require('session.socket.io') CookieParser = require("cookie-parser") -Settings = require "settings-sharelatex" - -logger = require "logger-sharelatex" -logger.initialize("real-time-sharelatex") - Metrics = require("metrics-sharelatex") Metrics.initialize(Settings.appName or "real-time") Metrics.event_loop.monitor(logger) -rclient = redis.createClient(Settings.redis.web) # Set up socket.io server app = express() @@ -42,14 +50,14 @@ io.configure -> app.get "/status", (req, res, next) -> res.send "real-time-sharelatex is alive" - + redisCheck = redis.activeHealthCheckRedis(Settings.redis.web) app.get "/health_check/redis", (req, res, next) -> if redisCheck.isAlive() res.send 200 else res.send 500 - + Router = require "./app/js/Router" Router.configure(app, io, sessionSockets) @@ -58,13 +66,13 @@ WebsocketLoadBalancer.listenForEditorEvents(io) DocumentUpdaterController = require "./app/js/DocumentUpdaterController" DocumentUpdaterController.listenForUpdatesFromDocumentUpdater(io) - + port = Settings.internal.realTime.port host = Settings.internal.realTime.host server.listen port, host, (error) -> throw error if error? logger.info "realtime starting up, listening on #{host}:#{port}" - + # Stop huge stack traces in logs from all the socket.io parsing steps. -Error.stackTraceLimit = 10 \ No newline at end of file +Error.stackTraceLimit = 10 diff --git a/services/real-time/package.json b/services/real-time/package.json index b0c7b46c2d..4f4034fe28 100644 --- a/services/real-time/package.json +++ b/services/real-time/package.json @@ -15,6 +15,7 @@ "cookie-parser": "^1.3.3", "express": "^4.10.1", "express-session": "^1.9.1", + "ioredis": "^2.4.0", "logger-sharelatex": "git+https://github.com/sharelatex/logger-sharelatex.git#v1.1.0", "metrics-sharelatex": "git+https://github.com/sharelatex/metrics-sharelatex.git#v1.4.0", "redis-sharelatex": "0.0.9", From de18231ef1500a67a3aff4d497dea5003a17aa1a Mon Sep 17 00:00:00 2001 From: Shane Kilkelly Date: Wed, 9 Nov 2016 12:09:15 +0000 Subject: [PATCH 068/491] clarify purpose of redis client --- services/real-time/app.coffee | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/services/real-time/app.coffee b/services/real-time/app.coffee index d21b133022..e4d1fd5446 100644 --- a/services/real-time/app.coffee +++ b/services/real-time/app.coffee @@ -11,9 +11,9 @@ redisSessionsSettings = Settings.redis.websessions or Settings.redis.web if redisSessionsSettings?.cluster? logger.log {}, "using redis cluster for web sessions" - rclient = new ioredis.Cluster(redisSessionsSettings.cluster) + sessionRedisClient = new ioredis.Cluster(redisSessionsSettings.cluster) else - rclient = redis.createClient(redisSessionsSettings) + sessionRedisClient = redis.createClient(redisSessionsSettings) RedisStore = require('connect-redis')(session) SessionSockets = require('session.socket.io') @@ -30,7 +30,7 @@ server = require('http').createServer(app) io = require('socket.io').listen(server) # Bind to sessions -sessionStore = new RedisStore(client: rclient) +sessionStore = new RedisStore(client: sessionRedisClient) cookieParser = CookieParser(Settings.security.sessionSecret) sessionSockets = new SessionSockets(io, sessionStore, cookieParser, Settings.cookieName) From 4cff89becc677d0d09a94c8fb11d433028ef5e7a Mon Sep 17 00:00:00 2001 From: James Allen Date: Thu, 8 Dec 2016 11:12:07 +0000 Subject: [PATCH 069/491] Fix acceptance tests --- .../app/coffee/WebsocketController.coffee | 2 +- .../coffee/ClientTrackingTests.coffee | 30 ++++++++++++++----- .../acceptance/coffee/JoinProjectTests.coffee | 3 +- 3 files changed, 24 insertions(+), 11 deletions(-) diff --git a/services/real-time/app/coffee/WebsocketController.coffee b/services/real-time/app/coffee/WebsocketController.coffee index d2aef5b424..ad7825f595 100644 --- a/services/real-time/app/coffee/WebsocketController.coffee +++ b/services/real-time/app/coffee/WebsocketController.coffee @@ -117,7 +117,7 @@ module.exports = WebsocketController = AuthorizationManager.assertClientCanViewProjectAndDoc client, cursorData.doc_id, (error) -> if error? - logger.warn {client_id: client.id, project_id, user_id}, "silently ignoring unauthorized updateClientPosition. Client likely hasn't called joinProject yet." + logger.warn {err: error, client_id: client.id, project_id, user_id}, "silently ignoring unauthorized updateClientPosition. Client likely hasn't called joinProject yet." return callback() cursorData.id = client.id cursorData.user_id = user_id if user_id? diff --git a/services/real-time/test/acceptance/coffee/ClientTrackingTests.coffee b/services/real-time/test/acceptance/coffee/ClientTrackingTests.coffee index a602df52d2..dc00035382 100644 --- a/services/real-time/test/acceptance/coffee/ClientTrackingTests.coffee +++ b/services/real-time/test/acceptance/coffee/ClientTrackingTests.coffee @@ -15,22 +15,28 @@ describe "clientTracking", -> (cb) => FixturesManager.setUpProject { privilegeLevel: "owner" - project: { name: "Test Project" } + project: { name: "Test Project" } }, (error, {@user_id, @project_id}) => cb() + (cb) => + FixturesManager.setUpDoc @project_id, {@lines, @version, @ops}, (e, {@doc_id}) => + cb(e) + (cb) => @clientA = RealTimeClient.connect() - @clientA.on "connect", cb + @clientA.on "connectionAccepted", cb (cb) => @clientB = RealTimeClient.connect() - - @clientB.on "connect", cb + @clientB.on "connectionAccepted", cb (cb) => @clientA.emit "joinProject", { project_id: @project_id }, cb + + (cb) => + @clientA.emit "joinDoc", @doc_id, cb (cb) => @clientB.emit "joinProject", { @@ -45,7 +51,7 @@ describe "clientTracking", -> @clientA.emit "clientTracking.updatePosition", { row: @row = 42 column: @column = 36 - doc_id: @doc_id = "mock-doc-id" + doc_id: @doc_id }, (error) -> throw error if error? setTimeout cb, 300 # Give the message a chance to reach client B. @@ -85,9 +91,14 @@ describe "clientTracking", -> publicAccess: "readAndWrite" }, (error, {@user_id, @project_id}) => cb() + (cb) => + FixturesManager.setUpDoc @project_id, {@lines, @version, @ops}, (e, {@doc_id}) => + cb(e) + (cb) => @clientA = RealTimeClient.connect() - @clientA.on "connect", cb + @clientA.on "connectionAccepted", cb + (cb) => @clientA.emit "joinProject", { project_id: @project_id @@ -98,12 +109,15 @@ describe "clientTracking", -> (cb) => @anonymous = RealTimeClient.connect() - @anonymous.on "connect", cb + @anonymous.on "connectionAccepted", cb (cb) => @anonymous.emit "joinProject", { project_id: @project_id }, cb + + (cb) => + @anonymous.emit "joinDoc", @doc_id, cb (cb) => @updates = [] @@ -113,7 +127,7 @@ describe "clientTracking", -> @anonymous.emit "clientTracking.updatePosition", { row: @row = 42 column: @column = 36 - doc_id: @doc_id = "mock-doc-id" + doc_id: @doc_id }, (error) -> throw error if error? setTimeout cb, 300 # Give the message a chance to reach client B. diff --git a/services/real-time/test/acceptance/coffee/JoinProjectTests.coffee b/services/real-time/test/acceptance/coffee/JoinProjectTests.coffee index 920330b395..91b0c7102f 100644 --- a/services/real-time/test/acceptance/coffee/JoinProjectTests.coffee +++ b/services/real-time/test/acceptance/coffee/JoinProjectTests.coffee @@ -83,8 +83,7 @@ describe "joinProject", -> ], done it "should return an error", -> - # We don't return specific errors - @error.message.should.equal "Something went wrong" + @error.message.should.equal "not authorized" it "should not have joined the project room", (done) -> RealTimeClient.getConnectedClient @client.socket.sessionid, (error, client) => From 9cf0eb554012f6a0d352a188765d01234f602c72 Mon Sep 17 00:00:00 2001 From: James Allen Date: Thu, 8 Dec 2016 11:14:27 +0000 Subject: [PATCH 070/491] Add in acceptance test script --- .../test/acceptance/scripts/full-test.sh | 23 +++++++++++++++++++ 1 file changed, 23 insertions(+) create mode 100755 services/real-time/test/acceptance/scripts/full-test.sh diff --git a/services/real-time/test/acceptance/scripts/full-test.sh b/services/real-time/test/acceptance/scripts/full-test.sh new file mode 100755 index 0000000000..9f6167e667 --- /dev/null +++ b/services/real-time/test/acceptance/scripts/full-test.sh @@ -0,0 +1,23 @@ +#! /usr/bin/env bash + +npm rebuild + +echo ">> Starting server..." + +grunt --no-color forever:app:start + +echo ">> Server started" + +sleep 5 + +echo ">> Running acceptance tests..." +grunt --no-color mochaTest:acceptance +_test_exit_code=$? + +echo ">> Killing server" + +grunt --no-color forever:app:stop + +echo ">> Done" + +exit $_test_exit_code From 5d377713d6267282d1da8996841599b693f78d34 Mon Sep 17 00:00:00 2001 From: James Allen Date: Thu, 8 Dec 2016 11:25:25 +0000 Subject: [PATCH 071/491] Try to fix issue with acceptance tests timing out --- .../acceptance/coffee/ApplyUpdateTests.coffee | 10 ++++++++-- .../test/acceptance/coffee/JoinDocTests.coffee | 16 ++++++++-------- .../acceptance/coffee/JoinProjectTests.coffee | 8 ++++---- .../test/acceptance/coffee/LeaveDocTests.coffee | 2 +- .../acceptance/coffee/LeaveProjectTests.coffee | 8 ++++---- .../acceptance/coffee/ReceiveUpdateTests.coffee | 4 ++-- 6 files changed, 27 insertions(+), 21 deletions(-) diff --git a/services/real-time/test/acceptance/coffee/ApplyUpdateTests.coffee b/services/real-time/test/acceptance/coffee/ApplyUpdateTests.coffee index b5676909cb..31d8dc2bc0 100644 --- a/services/real-time/test/acceptance/coffee/ApplyUpdateTests.coffee +++ b/services/real-time/test/acceptance/coffee/ApplyUpdateTests.coffee @@ -27,9 +27,12 @@ describe "applyOtUpdate", -> (cb) => FixturesManager.setUpDoc @project_id, {@lines, @version, @ops}, (e, {@doc_id}) => cb(e) - + (cb) => @client = RealTimeClient.connect() + @client.on "connectionAccepted", cb + + (cb) => @client.emit "joinProject", project_id: @project_id, cb (cb) => @@ -78,9 +81,12 @@ describe "applyOtUpdate", -> (cb) => FixturesManager.setUpDoc @project_id, {@lines, @version, @ops}, (e, {@doc_id}) => cb(e) - + (cb) => @client = RealTimeClient.connect() + @client.on "connectionAccepted", cb + + (cb) => @client.emit "joinProject", project_id: @project_id, cb (cb) => diff --git a/services/real-time/test/acceptance/coffee/JoinDocTests.coffee b/services/real-time/test/acceptance/coffee/JoinDocTests.coffee index ed26ea85cf..056909a627 100644 --- a/services/real-time/test/acceptance/coffee/JoinDocTests.coffee +++ b/services/real-time/test/acceptance/coffee/JoinDocTests.coffee @@ -26,10 +26,10 @@ describe "joinDoc", -> (cb) => FixturesManager.setUpDoc @project_id, {@lines, @version, @ops}, (e, {@doc_id}) => cb(e) - + (cb) => @client = RealTimeClient.connect() - @client.on "connect", cb + @client.on "connectionAccepted", cb (cb) => @client.emit "joinProject", project_id: @project_id, cb @@ -63,10 +63,10 @@ describe "joinDoc", -> (cb) => FixturesManager.setUpDoc @project_id, {@lines, @version, @ops}, (e, {@doc_id}) => cb(e) - + (cb) => @client = RealTimeClient.connect() - @client.on "connect", cb + @client.on "connectionAccepted", cb (cb) => @client.emit "joinProject", project_id: @project_id, cb @@ -100,10 +100,10 @@ describe "joinDoc", -> (cb) => FixturesManager.setUpDoc @project_id, {@lines, @version, @ops}, (e, {@doc_id}) => cb(e) - + (cb) => @client = RealTimeClient.connect() - @client.on "connect", cb + @client.on "connectionAccepted", cb (cb) => @client.emit "joinProject", project_id: @project_id, cb @@ -142,10 +142,10 @@ describe "joinDoc", -> (cb) => FixturesManager.setUpDoc @project_id, {@lines, @version, @ops}, (e, {@doc_id}) => cb(e) - + (cb) => @client = RealTimeClient.connect() - @client.on "connect", cb + @client.on "connectionAccepted", cb (cb) => @client.emit "joinProject", project_id: @project_id, cb diff --git a/services/real-time/test/acceptance/coffee/JoinProjectTests.coffee b/services/real-time/test/acceptance/coffee/JoinProjectTests.coffee index 91b0c7102f..471672f9ac 100644 --- a/services/real-time/test/acceptance/coffee/JoinProjectTests.coffee +++ b/services/real-time/test/acceptance/coffee/JoinProjectTests.coffee @@ -20,10 +20,10 @@ describe "joinProject", -> } }, (e, {@project_id, @user_id}) => cb(e) - + (cb) => @client = RealTimeClient.connect() - @client.on "connect", cb + @client.on "connectionAccepted", cb (cb) => @client.emit "joinProject", project_id: @project_id, (error, @project, @privilegeLevel, @protocolVersion) => @@ -72,10 +72,10 @@ describe "joinProject", -> } }, (e, {@project_id, @user_id}) => cb(e) - + (cb) => @client = RealTimeClient.connect() - @client.on "connect", cb + @client.on "connectionAccepted", cb (cb) => @client.emit "joinProject", project_id: @project_id, (@error, @project, @privilegeLevel, @protocolVersion) => diff --git a/services/real-time/test/acceptance/coffee/LeaveDocTests.coffee b/services/real-time/test/acceptance/coffee/LeaveDocTests.coffee index 8c676647d5..753bc79c62 100644 --- a/services/real-time/test/acceptance/coffee/LeaveDocTests.coffee +++ b/services/real-time/test/acceptance/coffee/LeaveDocTests.coffee @@ -29,7 +29,7 @@ describe "leaveDoc", -> (cb) => @client = RealTimeClient.connect() - @client.on "connect", cb + @client.on "connectionAccepted", cb (cb) => @client.emit "joinProject", project_id: @project_id, cb diff --git a/services/real-time/test/acceptance/coffee/LeaveProjectTests.coffee b/services/real-time/test/acceptance/coffee/LeaveProjectTests.coffee index 2d8c8c584f..7ef8b35c64 100644 --- a/services/real-time/test/acceptance/coffee/LeaveProjectTests.coffee +++ b/services/real-time/test/acceptance/coffee/LeaveProjectTests.coffee @@ -21,11 +21,11 @@ describe "leaveProject", -> (cb) => @clientA = RealTimeClient.connect() - @clientA.on "connect", cb + @clientA.on "connectionAccepted", cb (cb) => @clientB = RealTimeClient.connect() - @clientB.on "connect", cb + @clientB.on "connectionAccepted", cb @clientBDisconnectMessages = [] @clientB.on "clientTracking.clientDisconnected", (data) => @@ -46,7 +46,7 @@ describe "leaveProject", -> (cb) => # The API waits a little while before flushing changes - setTimeout done, require("../../../app/js/WebsocketController").FLUSH_IF_EMPTY_DELAY * 2 + setTimeout done, 1000 ], done @@ -91,7 +91,7 @@ describe "leaveProject", -> (cb) => # The API waits a little while before flushing changes - setTimeout done, require("../../../app/js/WebsocketController").FLUSH_IF_EMPTY_DELAY * 2 + setTimeout done, 1000 ], done it "should flush the project to the document updater", -> diff --git a/services/real-time/test/acceptance/coffee/ReceiveUpdateTests.coffee b/services/real-time/test/acceptance/coffee/ReceiveUpdateTests.coffee index fd1b9a0dfe..ec41598481 100644 --- a/services/real-time/test/acceptance/coffee/ReceiveUpdateTests.coffee +++ b/services/real-time/test/acceptance/coffee/ReceiveUpdateTests.coffee @@ -31,11 +31,11 @@ describe "receiveUpdate", -> (cb) => @clientA = RealTimeClient.connect() - @clientA.on "connect", cb + @clientA.on "connectionAccepted", cb (cb) => @clientB = RealTimeClient.connect() - @clientB.on "connect", cb + @clientB.on "connectionAccepted", cb (cb) => @clientA.emit "joinProject", { From bf2620ee0cb2939b643b174e0c6e09a46dbcd5ce Mon Sep 17 00:00:00 2001 From: James Allen Date: Thu, 8 Dec 2016 11:37:31 +0000 Subject: [PATCH 072/491] Return ranges from docupdater to client --- .../app/coffee/DocumentUpdaterManager.coffee | 2 +- .../app/coffee/WebsocketController.coffee | 4 +-- .../acceptance/coffee/JoinDocTests.coffee | 25 ++++++++++--------- .../coffee/helpers/FixturesManager.coffee | 4 +-- .../coffee/DocumentUpdaterManagerTests.coffee | 8 ++++-- .../coffee/WebsocketControllerTests.coffee | 7 +++--- 6 files changed, 28 insertions(+), 22 deletions(-) diff --git a/services/real-time/app/coffee/DocumentUpdaterManager.coffee b/services/real-time/app/coffee/DocumentUpdaterManager.coffee index 9e25585b1c..9d47a3ffc1 100644 --- a/services/real-time/app/coffee/DocumentUpdaterManager.coffee +++ b/services/real-time/app/coffee/DocumentUpdaterManager.coffee @@ -22,7 +22,7 @@ module.exports = DocumentUpdaterManager = body = JSON.parse(body) catch error return callback(error) - callback null, body?.lines, body?.version, body?.ops + callback null, body?.lines, body?.version, body?.ranges, body?.ops else if res.statusCode == 422 # Unprocessable Entity err = new Error("doc updater could not load requested ops") err.statusCode = res.statusCode diff --git a/services/real-time/app/coffee/WebsocketController.coffee b/services/real-time/app/coffee/WebsocketController.coffee index ad7825f595..c6aec7379d 100644 --- a/services/real-time/app/coffee/WebsocketController.coffee +++ b/services/real-time/app/coffee/WebsocketController.coffee @@ -79,7 +79,7 @@ module.exports = WebsocketController = AuthorizationManager.assertClientCanViewProject client, (error) -> return callback(error) if error? - DocumentUpdaterManager.getDocument project_id, doc_id, fromVersion, (error, lines, version, ops) -> + DocumentUpdaterManager.getDocument project_id, doc_id, fromVersion, (error, lines, version, ranges, ops) -> return callback(error) if error? # Encode any binary bits of data so it can go via WebSockets # See http://ecmanaut.blogspot.co.uk/2006/07/encoding-decoding-utf8-in-javascript.html @@ -93,7 +93,7 @@ module.exports = WebsocketController = escapedLines.push line AuthorizationManager.addAccessToDoc client, doc_id client.join(doc_id) - callback null, escapedLines, version, ops + callback null, escapedLines, version, ops, ranges logger.log {user_id, project_id, doc_id, fromVersion, client_id: client.id}, "client joined doc" leaveDoc: (client, doc_id, callback = (error) ->) -> diff --git a/services/real-time/test/acceptance/coffee/JoinDocTests.coffee b/services/real-time/test/acceptance/coffee/JoinDocTests.coffee index 056909a627..a9d5406345 100644 --- a/services/real-time/test/acceptance/coffee/JoinDocTests.coffee +++ b/services/real-time/test/acceptance/coffee/JoinDocTests.coffee @@ -13,6 +13,7 @@ describe "joinDoc", -> @lines = ["test", "doc", "lines"] @version = 42 @ops = ["mock", "doc", "ops"] + @ranges = {"mock": "ranges"} describe "when authorised readAndWrite", -> before (done) -> @@ -24,7 +25,7 @@ describe "joinDoc", -> cb(e) (cb) => - FixturesManager.setUpDoc @project_id, {@lines, @version, @ops}, (e, {@doc_id}) => + FixturesManager.setUpDoc @project_id, {@lines, @version, @ops, @ranges}, (e, {@doc_id}) => cb(e) (cb) => @@ -43,8 +44,8 @@ describe "joinDoc", -> .calledWith(@project_id, @doc_id, -1) .should.equal true - it "should return the doc lines, version and ops", -> - @returnedArgs.should.deep.equal [@lines, @version, @ops] + it "should return the doc lines, version, ranges and ops", -> + @returnedArgs.should.deep.equal [@lines, @version, @ops, @ranges] it "should have joined the doc room", (done) -> RealTimeClient.getConnectedClient @client.socket.sessionid, (error, client) => @@ -61,7 +62,7 @@ describe "joinDoc", -> cb(e) (cb) => - FixturesManager.setUpDoc @project_id, {@lines, @version, @ops}, (e, {@doc_id}) => + FixturesManager.setUpDoc @project_id, {@lines, @version, @ops, @ranges}, (e, {@doc_id}) => cb(e) (cb) => @@ -80,8 +81,8 @@ describe "joinDoc", -> .calledWith(@project_id, @doc_id, -1) .should.equal true - it "should return the doc lines, version and ops", -> - @returnedArgs.should.deep.equal [@lines, @version, @ops] + it "should return the doc lines, version, ranges and ops", -> + @returnedArgs.should.deep.equal [@lines, @version, @ops, @ranges] it "should have joined the doc room", (done) -> RealTimeClient.getConnectedClient @client.socket.sessionid, (error, client) => @@ -98,7 +99,7 @@ describe "joinDoc", -> cb(e) (cb) => - FixturesManager.setUpDoc @project_id, {@lines, @version, @ops}, (e, {@doc_id}) => + FixturesManager.setUpDoc @project_id, {@lines, @version, @ops, @ranges}, (e, {@doc_id}) => cb(e) (cb) => @@ -117,8 +118,8 @@ describe "joinDoc", -> .calledWith(@project_id, @doc_id, -1) .should.equal true - it "should return the doc lines, version and ops", -> - @returnedArgs.should.deep.equal [@lines, @version, @ops] + it "should return the doc lines, version, ranges and ops", -> + @returnedArgs.should.deep.equal [@lines, @version, @ops, @ranges] it "should have joined the doc room", (done) -> RealTimeClient.getConnectedClient @client.socket.sessionid, (error, client) => @@ -140,7 +141,7 @@ describe "joinDoc", -> cb(e) (cb) => - FixturesManager.setUpDoc @project_id, {@lines, @version, @ops}, (e, {@doc_id}) => + FixturesManager.setUpDoc @project_id, {@lines, @version, @ops, @ranges}, (e, {@doc_id}) => cb(e) (cb) => @@ -159,8 +160,8 @@ describe "joinDoc", -> .calledWith(@project_id, @doc_id, @fromVersion) .should.equal true - it "should return the doc lines, version and ops", -> - @returnedArgs.should.deep.equal [@lines, @version, @ops] + it "should return the doc lines, version, ranges and ops", -> + @returnedArgs.should.deep.equal [@lines, @version, @ops, @ranges] it "should have joined the doc room", (done) -> RealTimeClient.getConnectedClient @client.socket.sessionid, (error, client) => diff --git a/services/real-time/test/acceptance/coffee/helpers/FixturesManager.coffee b/services/real-time/test/acceptance/coffee/helpers/FixturesManager.coffee index 9ee5f4ef6f..0889b45c2a 100644 --- a/services/real-time/test/acceptance/coffee/helpers/FixturesManager.coffee +++ b/services/real-time/test/acceptance/coffee/helpers/FixturesManager.coffee @@ -32,9 +32,9 @@ module.exports = FixturesManager = options.lines ||= ["doc", "lines"] options.version ||= 42 options.ops ||= ["mock", "ops"] - {doc_id, lines, version, ops} = options + {doc_id, lines, version, ops, ranges} = options - MockDocUpdaterServer.createMockDoc project_id, doc_id, {lines, version, ops} + MockDocUpdaterServer.createMockDoc project_id, doc_id, {lines, version, ops, ranges} MockDocUpdaterServer.run (error) => throw error if error? callback null, {project_id, doc_id, lines, version, ops} diff --git a/services/real-time/test/unit/coffee/DocumentUpdaterManagerTests.coffee b/services/real-time/test/unit/coffee/DocumentUpdaterManagerTests.coffee index 8e75da1d0c..fbf44aeffe 100644 --- a/services/real-time/test/unit/coffee/DocumentUpdaterManagerTests.coffee +++ b/services/real-time/test/unit/coffee/DocumentUpdaterManagerTests.coffee @@ -20,6 +20,9 @@ describe 'DocumentUpdaterManager', -> 'logger-sharelatex': @logger = {log: sinon.stub(), error: sinon.stub(), warn: sinon.stub()} 'request': @request = {} 'redis-sharelatex' : createClient: () => @rclient + 'metrics-sharelatex': @Metrics = + Timer: class Timer + done: () -> describe "getDocument", -> beforeEach -> @@ -31,6 +34,7 @@ describe 'DocumentUpdaterManager', -> lines: @lines version: @version ops: @ops = ["mock-op-1", "mock-op-2"] + ranges: @ranges = {"mock": "ranges"} @fromVersion = 2 @request.get = sinon.stub().callsArgWith(1, null, {statusCode: 200}, @body) @DocumentUpdaterManager.getDocument @project_id, @doc_id, @fromVersion, @callback @@ -39,8 +43,8 @@ describe 'DocumentUpdaterManager', -> url = "#{@settings.apis.documentupdater.url}/project/#{@project_id}/doc/#{@doc_id}?fromVersion=#{@fromVersion}" @request.get.calledWith(url).should.equal true - it "should call the callback with the lines and version", -> - @callback.calledWith(null, @lines, @version, @ops).should.equal true + it "should call the callback with the lines, version, ranges and ops", -> + @callback.calledWith(null, @lines, @version, @ranges, @ops).should.equal true describe "when the document updater API returns an error", -> beforeEach -> diff --git a/services/real-time/test/unit/coffee/WebsocketControllerTests.coffee b/services/real-time/test/unit/coffee/WebsocketControllerTests.coffee index 48eccd4bee..7777327282 100644 --- a/services/real-time/test/unit/coffee/WebsocketControllerTests.coffee +++ b/services/real-time/test/unit/coffee/WebsocketControllerTests.coffee @@ -170,11 +170,12 @@ describe 'WebsocketController', -> @doc_lines = ["doc", "lines"] @version = 42 @ops = ["mock", "ops"] + @ranges = { "mock": "ranges" } @client.params.project_id = @project_id @AuthorizationManager.addAccessToDoc = sinon.stub() @AuthorizationManager.assertClientCanViewProject = sinon.stub().callsArgWith(1, null) - @DocumentUpdaterManager.getDocument = sinon.stub().callsArgWith(3, null, @doc_lines, @version, @ops) + @DocumentUpdaterManager.getDocument = sinon.stub().callsArgWith(3, null, @doc_lines, @version, @ranges, @ops) describe "with a fromVersion", -> beforeEach -> @@ -201,9 +202,9 @@ describe 'WebsocketController', -> .calledWith(@doc_id) .should.equal true - it "should call the callback with the lines, version and ops", -> + it "should call the callback with the lines, version, ranges and ops", -> @callback - .calledWith(null, @doc_lines, @version, @ops) + .calledWith(null, @doc_lines, @version, @ops, @ranges) .should.equal true it "should increment the join-doc metric", -> From e5160d9a36339cb1d04f6645fe82f7bb042fca3f Mon Sep 17 00:00:00 2001 From: Brian Gough Date: Mon, 21 Nov 2016 15:33:13 +0000 Subject: [PATCH 073/491] log client id when disconnecting on otUpdateError only log errors for connected clients --- services/real-time/app/coffee/DocumentUpdaterController.coffee | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/services/real-time/app/coffee/DocumentUpdaterController.coffee b/services/real-time/app/coffee/DocumentUpdaterController.coffee index 9ff6b8ee62..230eb14f98 100644 --- a/services/real-time/app/coffee/DocumentUpdaterController.coffee +++ b/services/real-time/app/coffee/DocumentUpdaterController.coffee @@ -35,8 +35,8 @@ module.exports = DocumentUpdaterController = client.emit "otUpdateApplied", update _processErrorFromDocumentUpdater: (io, doc_id, error, message) -> - logger.error err: error, doc_id: doc_id, "error from document updater" for client in io.sockets.clients(doc_id) + logger.error err: error, doc_id: doc_id, client_id: client.id, "error from document updater, disconnecting client" client.emit "otUpdateError", error, message client.disconnect() From 9fd099c24f04ffd04bba711b76988c6c60105d70 Mon Sep 17 00:00:00 2001 From: James Allen Date: Mon, 16 Jan 2017 17:09:44 +0100 Subject: [PATCH 074/491] Update getDoc signature to match reality --- services/real-time/app/coffee/WebsocketController.coffee | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/services/real-time/app/coffee/WebsocketController.coffee b/services/real-time/app/coffee/WebsocketController.coffee index c6aec7379d..0d9708dd5e 100644 --- a/services/real-time/app/coffee/WebsocketController.coffee +++ b/services/real-time/app/coffee/WebsocketController.coffee @@ -70,7 +70,7 @@ module.exports = WebsocketController = callback() , WebsocketController.FLUSH_IF_EMPTY_DELAY - joinDoc: (client, doc_id, fromVersion = -1, callback = (error, doclines, version, ops) ->) -> + joinDoc: (client, doc_id, fromVersion = -1, callback = (error, doclines, version, ops, ranges) ->) -> metrics.inc "editor.join-doc" Utils.getClientAttributes client, ["project_id", "user_id"], (error, {project_id, user_id}) -> return callback(error) if error? From 50930cd7b127d68f3e432fd1bc5321d6adaab743 Mon Sep 17 00:00:00 2001 From: Shane Kilkelly Date: Mon, 20 Feb 2017 10:14:41 +0000 Subject: [PATCH 075/491] Don't npm rebuild inside container --- services/real-time/test/acceptance/scripts/full-test.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/services/real-time/test/acceptance/scripts/full-test.sh b/services/real-time/test/acceptance/scripts/full-test.sh index 9f6167e667..8584cd17d0 100755 --- a/services/real-time/test/acceptance/scripts/full-test.sh +++ b/services/real-time/test/acceptance/scripts/full-test.sh @@ -1,6 +1,6 @@ #! /usr/bin/env bash -npm rebuild +# npm rebuild echo ">> Starting server..." From d468f662acbd10237079df7d5cbbd4f8aef6ea57 Mon Sep 17 00:00:00 2001 From: Brian Gough Date: Thu, 23 Feb 2017 12:04:36 +0000 Subject: [PATCH 076/491] handle disconnects of unauthenticated users --- .../app/coffee/WebsocketController.coffee | 10 ++++ .../coffee/WebsocketControllerTests.coffee | 51 ++++++++++++++++++- 2 files changed, 60 insertions(+), 1 deletion(-) diff --git a/services/real-time/app/coffee/WebsocketController.coffee b/services/real-time/app/coffee/WebsocketController.coffee index 0d9708dd5e..5c6f8eab53 100644 --- a/services/real-time/app/coffee/WebsocketController.coffee +++ b/services/real-time/app/coffee/WebsocketController.coffee @@ -52,6 +52,16 @@ module.exports = WebsocketController = metrics.inc "editor.leave-project" Utils.getClientAttributes client, ["project_id", "user_id"], (error, {project_id, user_id}) -> return callback(error) if error? + # bail out if the client had not managed to authenticate or join + # the project. Prevents downstream errors in docupdater from + # flushProjectToMongoAndDelete with null project_id. + if not user_id? + logger.log {client_id: client.id}, "client leaving, unknown user" + return callback() + if not project_id? + logger.log {user_id: user_id, client_id: client.id}, "client leaving, not in project" + return callback() + logger.log {project_id, user_id, client_id: client.id}, "client leaving project" WebsocketLoadBalancer.emitToRoom project_id, "clientTracking.clientDisconnected", client.id diff --git a/services/real-time/test/unit/coffee/WebsocketControllerTests.coffee b/services/real-time/test/unit/coffee/WebsocketControllerTests.coffee index 7777327282..33da6f2ba1 100644 --- a/services/real-time/test/unit/coffee/WebsocketControllerTests.coffee +++ b/services/real-time/test/unit/coffee/WebsocketControllerTests.coffee @@ -129,6 +129,7 @@ describe 'WebsocketController', -> throw "expected room_id to be project_id" return @clientsInRoom @client.params.project_id = @project_id + @client.params.user_id = @user_id @WebsocketController.FLUSH_IF_EMPTY_DELAY = 0 tk.reset() # Allow setTimeout to work. @@ -163,7 +164,55 @@ describe 'WebsocketController', -> it "should not flush the project in the document updater", -> @DocumentUpdaterManager.flushProjectToMongoAndDelete .called.should.equal false - + + describe "when client has not authenticated", -> + beforeEach (done) -> + @client.params.user_id = null + @client.params.project_id = null + @WebsocketController.leaveProject @io, @client, done + + it "should not end clientTracking.clientDisconnected to the project room", -> + @WebsocketLoadBalancer.emitToRoom + .calledWith(@project_id, "clientTracking.clientDisconnected", @client.id) + .should.equal false + + it "should not mark the user as disconnected", -> + @ConnectedUsersManager.markUserAsDisconnected + .calledWith(@project_id, @client.id) + .should.equal false + + it "should not flush the project in the document updater", -> + @DocumentUpdaterManager.flushProjectToMongoAndDelete + .calledWith(@project_id) + .should.equal false + + it "should increment the leave-project metric", -> + @metrics.inc.calledWith("editor.leave-project").should.equal true + + describe "when client has not joined a project", -> + beforeEach (done) -> + @client.params.user_id = @user_id + @client.params.project_id = null + @WebsocketController.leaveProject @io, @client, done + + it "should not end clientTracking.clientDisconnected to the project room", -> + @WebsocketLoadBalancer.emitToRoom + .calledWith(@project_id, "clientTracking.clientDisconnected", @client.id) + .should.equal false + + it "should not mark the user as disconnected", -> + @ConnectedUsersManager.markUserAsDisconnected + .calledWith(@project_id, @client.id) + .should.equal false + + it "should not flush the project in the document updater", -> + @DocumentUpdaterManager.flushProjectToMongoAndDelete + .calledWith(@project_id) + .should.equal false + + it "should increment the leave-project metric", -> + @metrics.inc.calledWith("editor.leave-project").should.equal true + describe "joinDoc", -> beforeEach -> @doc_id = "doc-id-123" From d939f6cd65e8623a99bcc0fcae07d82c13d228af Mon Sep 17 00:00:00 2001 From: James Allen Date: Wed, 15 Mar 2017 15:45:18 +0000 Subject: [PATCH 077/491] Remove some old logging --- services/real-time/app/coffee/WebsocketController.coffee | 9 +-------- 1 file changed, 1 insertion(+), 8 deletions(-) diff --git a/services/real-time/app/coffee/WebsocketController.coffee b/services/real-time/app/coffee/WebsocketController.coffee index 0d9708dd5e..37fe0ed511 100644 --- a/services/real-time/app/coffee/WebsocketController.coffee +++ b/services/real-time/app/coffee/WebsocketController.coffee @@ -154,16 +154,10 @@ module.exports = WebsocketController = applyOtUpdate: (client, doc_id, update, callback = (error) ->) -> - cbc_0 = 0 # Callback counter - cbc_1 = 0 Utils.getClientAttributes client, ["user_id", "project_id"], (error, {user_id, project_id}) -> - cbc_0++ return callback(error) if error? return callback(new Error("no project_id found on client")) if !project_id? - # Omit this logging for now since it's likely too noisey - #logger.log {user_id, project_id, doc_id, client_id: client.id, update: update}, "applying update" AuthorizationManager.assertClientCanEditProjectAndDoc client, doc_id, (error) -> - cbc_1++ if error? logger.error {err: error, doc_id, client_id: client.id, version: update.v}, "client is not authorized to make update" setTimeout () -> @@ -178,11 +172,10 @@ module.exports = WebsocketController = metrics.set "editor.active-projects", project_id, 0.3 metrics.set "editor.active-users", user_id, 0.3 - logger.log {user_id, doc_id, project_id, client_id: client.id, version: update.v, cbc_0, cbc_1}, "sending update to doc updater" + logger.log {user_id, doc_id, project_id, client_id: client.id, version: update.v}, "sending update to doc updater" DocumentUpdaterManager.queueChange project_id, doc_id, update, (error) -> if error? logger.error {err: error, project_id, doc_id, client_id: client.id, version: update.v}, "document was not available for update" client.disconnect() callback(error) - #logger.log {user_id, project_id, doc_id, client_id: client.id}, "applied update" From 8766646149bada31e6795cd112dfbc2934b24bd8 Mon Sep 17 00:00:00 2001 From: James Allen Date: Wed, 15 Mar 2017 15:45:52 +0000 Subject: [PATCH 078/491] Allow users to send a comment update if they are read-only --- .../app/coffee/WebsocketController.coffee | 19 +++++- .../acceptance/coffee/ApplyUpdateTests.coffee | 61 ++++++++++++++++++- .../coffee/WebsocketControllerTests.coffee | 43 ++++++++++++- 3 files changed, 118 insertions(+), 5 deletions(-) diff --git a/services/real-time/app/coffee/WebsocketController.coffee b/services/real-time/app/coffee/WebsocketController.coffee index 37fe0ed511..bf893af78a 100644 --- a/services/real-time/app/coffee/WebsocketController.coffee +++ b/services/real-time/app/coffee/WebsocketController.coffee @@ -157,7 +157,7 @@ module.exports = WebsocketController = Utils.getClientAttributes client, ["user_id", "project_id"], (error, {user_id, project_id}) -> return callback(error) if error? return callback(new Error("no project_id found on client")) if !project_id? - AuthorizationManager.assertClientCanEditProjectAndDoc client, doc_id, (error) -> + WebsocketController._assertClientCanApplyUpdate client, doc_id, update, (error) -> if error? logger.error {err: error, doc_id, client_id: client.id, version: update.v}, "client is not authorized to make update" setTimeout () -> @@ -179,3 +179,20 @@ module.exports = WebsocketController = logger.error {err: error, project_id, doc_id, client_id: client.id, version: update.v}, "document was not available for update" client.disconnect() callback(error) + + _assertClientCanApplyUpdate: (client, doc_id, update, callback) -> + AuthorizationManager.assertClientCanEditProjectAndDoc client, doc_id, (error) -> + if error? + if error.message == "not authorized" and WebsocketController._isCommentUpdate(update) + # This might be a comment op, which we only need read-only priveleges for + AuthorizationManager.assertClientCanViewProjectAndDoc client, doc_id, callback + else + return callback(error) + else + return callback(null) + + _isCommentUpdate: (update) -> + for op in update.op + if !op.c? + return false + return true \ No newline at end of file diff --git a/services/real-time/test/acceptance/coffee/ApplyUpdateTests.coffee b/services/real-time/test/acceptance/coffee/ApplyUpdateTests.coffee index 31d8dc2bc0..bb2df04ae2 100644 --- a/services/real-time/test/acceptance/coffee/ApplyUpdateTests.coffee +++ b/services/real-time/test/acceptance/coffee/ApplyUpdateTests.coffee @@ -69,7 +69,7 @@ describe "applyOtUpdate", -> (cb) => rclient.del "PendingUpdates:#{@doc_id}", cb ], done - describe "when not authorized", -> + describe "when authorized to read-only with an edit update", -> before (done) -> async.series [ (cb) => @@ -109,4 +109,61 @@ describe "applyOtUpdate", -> it "should not put the update in redis", (done) -> rclient.llen "PendingUpdates:#{@doc_id}", (error, len) => len.should.equal 0 - done() \ No newline at end of file + done() + + describe "when authorized to read-only with a comment update", -> + before (done) -> + @comment_update = { + op: [{c: "foo", p: 42}] + } + async.series [ + (cb) => + FixturesManager.setUpProject { + privilegeLevel: "readOnly" + }, (e, {@project_id, @user_id}) => + cb(e) + + (cb) => + FixturesManager.setUpDoc @project_id, {@lines, @version, @ops}, (e, {@doc_id}) => + cb(e) + + (cb) => + @client = RealTimeClient.connect() + @client.on "connectionAccepted", cb + + (cb) => + @client.emit "joinProject", project_id: @project_id, cb + + (cb) => + @client.emit "joinDoc", @doc_id, cb + + (cb) => + @client.emit "applyOtUpdate", @doc_id, @comment_update, cb + ], done + + it "should push the doc into the pending updates list", (done) -> + rclient.lrange "pending-updates-list", 0, -1, (error, [doc_id]) => + doc_id.should.equal "#{@project_id}:#{@doc_id}" + done() + + it "should add the doc to the pending updates set in redis", (done) -> + rclient.sismember "DocsWithPendingUpdates", "#{@project_id}:#{@doc_id}", (error, isMember) => + isMember.should.equal 1 + done() + + it "should push the update into redis", (done) -> + rclient.lrange "PendingUpdates:#{@doc_id}", 0, -1, (error, [update]) => + update = JSON.parse(update) + update.op.should.deep.equal @comment_update.op + update.meta.should.deep.equal { + source: @client.socket.sessionid + user_id: @user_id + } + done() + + after (done) -> + async.series [ + (cb) => rclient.del "pending-updates-list", cb + (cb) => rclient.del "DocsWithPendingUpdates", "#{@project_id}:#{@doc_id}", cb + (cb) => rclient.del "PendingUpdates:#{@doc_id}", cb + ], done \ No newline at end of file diff --git a/services/real-time/test/unit/coffee/WebsocketControllerTests.coffee b/services/real-time/test/unit/coffee/WebsocketControllerTests.coffee index 7777327282..534a999ee8 100644 --- a/services/real-time/test/unit/coffee/WebsocketControllerTests.coffee +++ b/services/real-time/test/unit/coffee/WebsocketControllerTests.coffee @@ -1,6 +1,7 @@ chai = require('chai') should = chai.should() sinon = require("sinon") +expect = chai.expect modulePath = "../../../app/js/WebsocketController.js" SandboxedModule = require('sandboxed-module') tk = require "timekeeper" @@ -368,7 +369,7 @@ describe 'WebsocketController', -> @update = {op: {p: 12, t: "foo"}} @client.params.user_id = @user_id @client.params.project_id = @project_id - @AuthorizationManager.assertClientCanEditProjectAndDoc = sinon.stub().callsArg(2) + @WebsocketController._assertClientCanApplyUpdate = sinon.stub().yields() @DocumentUpdaterManager.queueChange = sinon.stub().callsArg(3) describe "succesfully", -> @@ -416,7 +417,7 @@ describe 'WebsocketController', -> describe "when not authorized", -> beforeEach -> @client.disconnect = sinon.stub() - @AuthorizationManager.assertClientCanEditProjectAndDoc = sinon.stub().callsArgWith(2, @error = new Error("not authorized")) + @WebsocketController._assertClientCanApplyUpdate = sinon.stub().yields(@error = new Error("not authorized")) @WebsocketController.applyOtUpdate @client, @doc_id, @update, @callback # This happens in a setTimeout to allow the client a chance to receive the error first. @@ -429,3 +430,41 @@ describe 'WebsocketController', -> it "should call the callback with the error", -> @callback.calledWith(@error).should.equal true + + describe "_assertClientCanApplyUpdate", -> + beforeEach -> + @edit_update = { op: [{i: "foo", p: 42}, {c: "bar", p: 132}] } # comments may still be in an edit op + @comment_update = { op: [{c: "bar", p: 132}] } + @AuthorizationManager.assertClientCanEditProjectAndDoc = sinon.stub() + @AuthorizationManager.assertClientCanViewProjectAndDoc = sinon.stub() + + describe "with a read-write client", -> + it "should return successfully", (done) -> + @AuthorizationManager.assertClientCanEditProjectAndDoc.yields(null) + @WebsocketController._assertClientCanApplyUpdate @client, @doc_id, @edit_update, (error) -> + expect(error).to.be.null + done() + + describe "with a read-only client and an edit op", -> + it "should return an error", (done) -> + @AuthorizationManager.assertClientCanEditProjectAndDoc.yields(new Error("not authorized")) + @AuthorizationManager.assertClientCanViewProjectAndDoc.yields(null) + @WebsocketController._assertClientCanApplyUpdate @client, @doc_id, @edit_update, (error) -> + expect(error.message).to.equal "not authorized" + done() + + describe "with a read-only client and a comment op", -> + it "should return successfully", (done) -> + @AuthorizationManager.assertClientCanEditProjectAndDoc.yields(new Error("not authorized")) + @AuthorizationManager.assertClientCanViewProjectAndDoc.yields(null) + @WebsocketController._assertClientCanApplyUpdate @client, @doc_id, @comment_update, (error) -> + expect(error).to.be.null + done() + + describe "with a totally unauthorized client", -> + it "should return an error", (done) -> + @AuthorizationManager.assertClientCanEditProjectAndDoc.yields(new Error("not authorized")) + @AuthorizationManager.assertClientCanViewProjectAndDoc.yields(new Error("not authorized")) + @WebsocketController._assertClientCanApplyUpdate @client, @doc_id, @comment_update, (error) -> + expect(error.message).to.equal "not authorized" + done() From ed76d57bf8a8b2b018e87559c0a664ee34eda75d Mon Sep 17 00:00:00 2001 From: Shane Kilkelly Date: Mon, 27 Mar 2017 14:51:46 +0100 Subject: [PATCH 079/491] Add a .nvmrc file --- services/real-time/.nvmrc | 1 + 1 file changed, 1 insertion(+) create mode 100644 services/real-time/.nvmrc diff --git a/services/real-time/.nvmrc b/services/real-time/.nvmrc new file mode 100644 index 0000000000..d87edbfc10 --- /dev/null +++ b/services/real-time/.nvmrc @@ -0,0 +1 @@ +4.2.1 \ No newline at end of file From 720f24427ab87b20da796f448e0686802fcb6c0b Mon Sep 17 00:00:00 2001 From: James Allen Date: Tue, 2 May 2017 15:51:17 +0100 Subject: [PATCH 080/491] Use new redis-sharelatex with support for cluster --- services/real-time/app.coffee | 12 ++++--- .../app/coffee/ConnectedUsersManager.coffee | 36 +++++++++---------- .../coffee/DocumentUpdaterController.coffee | 2 +- .../app/coffee/DocumentUpdaterManager.coffee | 6 ++-- .../app/coffee/WebsocketLoadBalancer.coffee | 4 +-- .../real-time/config/settings.defaults.coffee | 12 ++++++- services/real-time/package.json | 2 +- .../coffee/ConnectedUsersManagerTests.coffee | 5 ++- .../DocumentUpdaterControllerTests.coffee | 9 +++-- .../coffee/DocumentUpdaterManagerTests.coffee | 4 ++- .../coffee/WebsocketLoadBalancerTests.coffee | 4 +-- 11 files changed, 56 insertions(+), 40 deletions(-) diff --git a/services/real-time/app.coffee b/services/real-time/app.coffee index e4d1fd5446..578520296d 100644 --- a/services/real-time/app.coffee +++ b/services/real-time/app.coffee @@ -51,12 +51,14 @@ io.configure -> app.get "/status", (req, res, next) -> res.send "real-time-sharelatex is alive" -redisCheck = redis.activeHealthCheckRedis(Settings.redis.web) +rclient = require("redis-sharelatex").createClient(Settings.redis.realtime) app.get "/health_check/redis", (req, res, next) -> - if redisCheck.isAlive() - res.send 200 - else - res.send 500 + rclient.healthCheck (error) -> + if error? + logger.err {err: error}, "failed redis health check" + res.sendStatus 500 + else + res.sendStatus 200 Router = require "./app/js/Router" Router.configure(app, io, sessionSockets) diff --git a/services/real-time/app/coffee/ConnectedUsersManager.coffee b/services/real-time/app/coffee/ConnectedUsersManager.coffee index fa7e366ad8..46454471ba 100644 --- a/services/real-time/app/coffee/ConnectedUsersManager.coffee +++ b/services/real-time/app/coffee/ConnectedUsersManager.coffee @@ -2,8 +2,8 @@ async = require("async") Settings = require('settings-sharelatex') logger = require("logger-sharelatex") redis = require("redis-sharelatex") -rclient = redis.createClient(Settings.redis.web) - +rclient = redis.createClient(Settings.redis.realtime) +Keys = Settings.redis.realtime.key_schema ONE_HOUR_IN_S = 60 * 60 ONE_DAY_IN_S = ONE_HOUR_IN_S * 24 @@ -11,10 +11,6 @@ FOUR_DAYS_IN_S = ONE_DAY_IN_S * 4 USER_TIMEOUT_IN_S = ONE_HOUR_IN_S / 4 -buildProjectSetKey = (project_id)-> return "clients_in_project:#{project_id}" -buildUserKey = (project_id, client_id)-> return "connected_user:#{project_id}:#{client_id}" - - module.exports = # Use the same method for when a user connects, and when a user sends a cursor @@ -25,18 +21,18 @@ module.exports = multi = rclient.multi() - multi.sadd buildProjectSetKey(project_id), client_id - multi.expire buildProjectSetKey(project_id), FOUR_DAYS_IN_S + multi.sadd Keys.clientsInProject({project_id}), client_id + multi.expire Keys.clientsInProject({project_id}), FOUR_DAYS_IN_S - multi.hset buildUserKey(project_id, client_id), "last_updated_at", Date.now() - multi.hset buildUserKey(project_id, client_id), "user_id", user._id - multi.hset buildUserKey(project_id, client_id), "first_name", user.first_name or "" - multi.hset buildUserKey(project_id, client_id), "last_name", user.last_name or "" - multi.hset buildUserKey(project_id, client_id), "email", user.email or "" + multi.hset Keys.connectedUser({project_id, client_id}), "last_updated_at", Date.now() + multi.hset Keys.connectedUser({project_id, client_id}), "user_id", user._id + multi.hset Keys.connectedUser({project_id, client_id}), "first_name", user.first_name or "" + multi.hset Keys.connectedUser({project_id, client_id}), "last_name", user.last_name or "" + multi.hset Keys.connectedUser({project_id, client_id}), "email", user.email or "" if cursorData? - multi.hset buildUserKey(project_id, client_id), "cursorData", JSON.stringify(cursorData) - multi.expire buildUserKey(project_id, client_id), USER_TIMEOUT_IN_S + multi.hset Keys.connectedUser({project_id, client_id}), "cursorData", JSON.stringify(cursorData) + multi.expire Keys.connectedUser({project_id, client_id}), USER_TIMEOUT_IN_S multi.exec (err)-> if err? @@ -46,14 +42,14 @@ module.exports = markUserAsDisconnected: (project_id, client_id, callback)-> logger.log project_id:project_id, client_id:client_id, "marking user as disconnected" multi = rclient.multi() - multi.srem buildProjectSetKey(project_id), client_id - multi.expire buildProjectSetKey(project_id), FOUR_DAYS_IN_S - multi.del buildUserKey(project_id, client_id) + multi.srem Keys.clientsInProject({project_id}), client_id + multi.expire Keys.clientsInProject({project_id}), FOUR_DAYS_IN_S + multi.del Keys.connectedUser({project_id, client_id}) multi.exec callback _getConnectedUser: (project_id, client_id, callback)-> - rclient.hgetall buildUserKey(project_id, client_id), (err, result)-> + rclient.hgetall Keys.connectedUser({project_id, client_id}), (err, result)-> if !result? result = connected : false @@ -71,7 +67,7 @@ module.exports = getConnectedUsers: (project_id, callback)-> self = @ - rclient.smembers buildProjectSetKey(project_id), (err, results)-> + rclient.smembers Keys.clientsInProject({project_id}), (err, results)-> return callback(err) if err? jobs = results.map (client_id)-> (cb)-> diff --git a/services/real-time/app/coffee/DocumentUpdaterController.coffee b/services/real-time/app/coffee/DocumentUpdaterController.coffee index 230eb14f98..01a8732a2a 100644 --- a/services/real-time/app/coffee/DocumentUpdaterController.coffee +++ b/services/real-time/app/coffee/DocumentUpdaterController.coffee @@ -1,7 +1,7 @@ logger = require "logger-sharelatex" settings = require 'settings-sharelatex' redis = require("redis-sharelatex") -rclient = redis.createClient(settings.redis.web) +rclient = redis.createClient(settings.redis.documentupdater) SafeJsonParse = require "./SafeJsonParse" MESSAGE_SIZE_LOG_LIMIT = 1024 * 1024 # 1Mb diff --git a/services/real-time/app/coffee/DocumentUpdaterManager.coffee b/services/real-time/app/coffee/DocumentUpdaterManager.coffee index 9d47a3ffc1..0c8d864ce8 100644 --- a/services/real-time/app/coffee/DocumentUpdaterManager.coffee +++ b/services/real-time/app/coffee/DocumentUpdaterManager.coffee @@ -3,8 +3,8 @@ logger = require "logger-sharelatex" settings = require "settings-sharelatex" metrics = require("metrics-sharelatex") -redis = require("redis-sharelatex") -rclient = redis.createClient(settings.redis.web) +rclient = require("redis-sharelatex").createClient(settings.redis.documentupdater) +Keys = settings.redis.documentupdater.key_schema module.exports = DocumentUpdaterManager = getDocument: (project_id, doc_id, fromVersion, callback = (error, exists, doclines, version) ->) -> @@ -56,7 +56,7 @@ module.exports = DocumentUpdaterManager = jsonChange = JSON.stringify change doc_key = "#{project_id}:#{doc_id}" multi = rclient.multi() - multi.rpush "PendingUpdates:#{doc_id}", jsonChange + multi.rpush Keys.pendingUpdates({doc_id}), jsonChange multi.sadd "DocsWithPendingUpdates", doc_key multi.rpush "pending-updates-list", doc_key multi.exec (error) -> diff --git a/services/real-time/app/coffee/WebsocketLoadBalancer.coffee b/services/real-time/app/coffee/WebsocketLoadBalancer.coffee index d346c41107..56ce8e5d30 100644 --- a/services/real-time/app/coffee/WebsocketLoadBalancer.coffee +++ b/services/real-time/app/coffee/WebsocketLoadBalancer.coffee @@ -2,8 +2,8 @@ Settings = require 'settings-sharelatex' logger = require 'logger-sharelatex' redis = require("redis-sharelatex") SafeJsonParse = require "./SafeJsonParse" -rclientPub = redis.createClient(Settings.redis.web) -rclientSub = redis.createClient(Settings.redis.web) +rclientPub = redis.createClient(Settings.redis.realtime) +rclientSub = redis.createClient(Settings.redis.realtime) module.exports = WebsocketLoadBalancer = rclientPub: rclientPub diff --git a/services/real-time/config/settings.defaults.coffee b/services/real-time/config/settings.defaults.coffee index 5e6a3691ab..b8fe111b98 100644 --- a/services/real-time/config/settings.defaults.coffee +++ b/services/real-time/config/settings.defaults.coffee @@ -1,9 +1,19 @@ module.exports = redis: - web: + realtime: host: "localhost" port: "6379" password: "" + key_schema: + clientsInProject: ({project_id}) -> "clients_in_project:#{project_id}" + connectedUser: ({project_id, client_id})-> "connected_user:#{project_id}:#{client_id}" + + documentupdater: + host: "localhost" + port: "6379" + password: "" + key_schema: + pendingUpdates: ({doc_id}) -> "PendingUpdates:#{doc_id}" internal: realTime: diff --git a/services/real-time/package.json b/services/real-time/package.json index 4f4034fe28..fa67f0511e 100644 --- a/services/real-time/package.json +++ b/services/real-time/package.json @@ -18,7 +18,7 @@ "ioredis": "^2.4.0", "logger-sharelatex": "git+https://github.com/sharelatex/logger-sharelatex.git#v1.1.0", "metrics-sharelatex": "git+https://github.com/sharelatex/metrics-sharelatex.git#v1.4.0", - "redis-sharelatex": "0.0.9", + "redis-sharelatex": "git+https://github.com/sharelatex/redis-sharelatex.git#v1.0.0", "request": "~2.34.0", "session.socket.io": "^0.1.6", "settings-sharelatex": "git+https://github.com/sharelatex/settings-sharelatex.git#v1.0.0", diff --git a/services/real-time/test/unit/coffee/ConnectedUsersManagerTests.coffee b/services/real-time/test/unit/coffee/ConnectedUsersManagerTests.coffee index 982f4a170f..9f77cb0e8b 100644 --- a/services/real-time/test/unit/coffee/ConnectedUsersManagerTests.coffee +++ b/services/real-time/test/unit/coffee/ConnectedUsersManagerTests.coffee @@ -15,7 +15,10 @@ describe "ConnectedUsersManager", -> @settings = redis: - web:{} + realtime: + key_schema: + clientsInProject: ({project_id}) -> "clients_in_project:#{project_id}" + connectedUser: ({project_id, client_id})-> "connected_user:#{project_id}:#{client_id}" @rClient = auth:-> setex:sinon.stub() diff --git a/services/real-time/test/unit/coffee/DocumentUpdaterControllerTests.coffee b/services/real-time/test/unit/coffee/DocumentUpdaterControllerTests.coffee index 8673b12b41..cb77a11513 100644 --- a/services/real-time/test/unit/coffee/DocumentUpdaterControllerTests.coffee +++ b/services/real-time/test/unit/coffee/DocumentUpdaterControllerTests.coffee @@ -13,10 +13,13 @@ describe "DocumentUpdaterController", -> @EditorUpdatesController = SandboxedModule.require modulePath, requires: "logger-sharelatex": @logger = { error: sinon.stub(), log: sinon.stub() } "settings-sharelatex": @settings = - redis: web: {} + redis: + documentupdater: + key_schema: + pendingUpdates: ({doc_id}) -> "PendingUpdates:#{doc_id}" "redis-sharelatex" : - createClient: ()=> - @rclient = {auth:->} + createClient: () => + @rclient = {} "./SafeJsonParse": @SafeJsonParse = parse: (data, cb) => cb null, JSON.parse(data) diff --git a/services/real-time/test/unit/coffee/DocumentUpdaterManagerTests.coffee b/services/real-time/test/unit/coffee/DocumentUpdaterManagerTests.coffee index fbf44aeffe..c693880e39 100644 --- a/services/real-time/test/unit/coffee/DocumentUpdaterManagerTests.coffee +++ b/services/real-time/test/unit/coffee/DocumentUpdaterManagerTests.coffee @@ -12,7 +12,9 @@ describe 'DocumentUpdaterManager', -> @version = 42 @settings = apis: documentupdater: url: "http://doc-updater.example.com" - redis: web: {} + redis: documentupdater: + key_schema: + pendingUpdates: ({doc_id}) -> "PendingUpdates:#{doc_id}" @rclient = {auth:->} @DocumentUpdaterManager = SandboxedModule.require modulePath, requires: diff --git a/services/real-time/test/unit/coffee/WebsocketLoadBalancerTests.coffee b/services/real-time/test/unit/coffee/WebsocketLoadBalancerTests.coffee index 07afe988ab..5cae81a31d 100644 --- a/services/real-time/test/unit/coffee/WebsocketLoadBalancerTests.coffee +++ b/services/real-time/test/unit/coffee/WebsocketLoadBalancerTests.coffee @@ -5,10 +5,10 @@ modulePath = require('path').join __dirname, '../../../app/js/WebsocketLoadBalan describe "WebsocketLoadBalancer", -> beforeEach -> + @rclient = {} @WebsocketLoadBalancer = SandboxedModule.require modulePath, requires: "redis-sharelatex": - createClient: () -> - auth:-> + createClient: () => @rclient "logger-sharelatex": @logger = { log: sinon.stub(), error: sinon.stub() } "./SafeJsonParse": @SafeJsonParse = parse: (data, cb) => cb null, JSON.parse(data) From d04be1c000d5d399d31525c2604e87609741cd82 Mon Sep 17 00:00:00 2001 From: James Allen Date: Tue, 9 May 2017 17:09:00 +0100 Subject: [PATCH 081/491] Update redis-sharelatex --- services/real-time/package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/services/real-time/package.json b/services/real-time/package.json index fa67f0511e..4b3de3d5f3 100644 --- a/services/real-time/package.json +++ b/services/real-time/package.json @@ -18,7 +18,7 @@ "ioredis": "^2.4.0", "logger-sharelatex": "git+https://github.com/sharelatex/logger-sharelatex.git#v1.1.0", "metrics-sharelatex": "git+https://github.com/sharelatex/metrics-sharelatex.git#v1.4.0", - "redis-sharelatex": "git+https://github.com/sharelatex/redis-sharelatex.git#v1.0.0", + "redis-sharelatex": "git+https://github.com/sharelatex/redis-sharelatex.git#v1.0.2", "request": "~2.34.0", "session.socket.io": "^0.1.6", "settings-sharelatex": "git+https://github.com/sharelatex/settings-sharelatex.git#v1.0.0", From 01d0b63f2c47f58705048a503ab023fdd71e6df0 Mon Sep 17 00:00:00 2001 From: James Allen Date: Wed, 10 May 2017 15:52:35 +0100 Subject: [PATCH 082/491] Update config for websessions --- services/real-time/app.coffee | 9 +-------- services/real-time/config/settings.defaults.coffee | 7 ++++++- 2 files changed, 7 insertions(+), 9 deletions(-) diff --git a/services/real-time/app.coffee b/services/real-time/app.coffee index 578520296d..0365f773c6 100644 --- a/services/real-time/app.coffee +++ b/services/real-time/app.coffee @@ -4,16 +4,9 @@ logger.initialize("real-time-sharelatex") express = require("express") session = require("express-session") redis = require("redis-sharelatex") -ioredis = require('ioredis') Settings = require "settings-sharelatex" -redisSessionsSettings = Settings.redis.websessions or Settings.redis.web - -if redisSessionsSettings?.cluster? - logger.log {}, "using redis cluster for web sessions" - sessionRedisClient = new ioredis.Cluster(redisSessionsSettings.cluster) -else - sessionRedisClient = redis.createClient(redisSessionsSettings) +sessionRedisClient = redis.createClient(Settings.redis.websessions) RedisStore = require('connect-redis')(session) SessionSockets = require('session.socket.io') diff --git a/services/real-time/config/settings.defaults.coffee b/services/real-time/config/settings.defaults.coffee index b8fe111b98..d939dd55a8 100644 --- a/services/real-time/config/settings.defaults.coffee +++ b/services/real-time/config/settings.defaults.coffee @@ -14,7 +14,12 @@ module.exports = password: "" key_schema: pendingUpdates: ({doc_id}) -> "PendingUpdates:#{doc_id}" - + + websessions: + host: "localhost" + port: "6379" + password: "" + internal: realTime: port: 3026 From a8917b933f9840f7e0f828dbc7d3586d3c34d578 Mon Sep 17 00:00:00 2001 From: James Allen Date: Thu, 11 May 2017 17:27:28 +0100 Subject: [PATCH 083/491] Remove multi call to make compatible with redis-cluster --- .../app/coffee/DocumentUpdaterManager.coffee | 10 ++++------ .../test/acceptance/coffee/ApplyUpdateTests.coffee | 10 ---------- .../unit/coffee/DocumentUpdaterManagerTests.coffee | 14 +++----------- 3 files changed, 7 insertions(+), 27 deletions(-) diff --git a/services/real-time/app/coffee/DocumentUpdaterManager.coffee b/services/real-time/app/coffee/DocumentUpdaterManager.coffee index 0c8d864ce8..ab893aa47f 100644 --- a/services/real-time/app/coffee/DocumentUpdaterManager.coffee +++ b/services/real-time/app/coffee/DocumentUpdaterManager.coffee @@ -55,10 +55,8 @@ module.exports = DocumentUpdaterManager = queueChange: (project_id, doc_id, change, callback = ()->)-> jsonChange = JSON.stringify change doc_key = "#{project_id}:#{doc_id}" - multi = rclient.multi() - multi.rpush Keys.pendingUpdates({doc_id}), jsonChange - multi.sadd "DocsWithPendingUpdates", doc_key - multi.rpush "pending-updates-list", doc_key - multi.exec (error) -> + # Push onto pendingUpdates for doc_id first, because once the doc updater + # gets an entry on pending-updates-list, it starts processing. + rclient.rpush Keys.pendingUpdates({doc_id}), jsonChange, (error) -> return callback(error) if error? - callback() + rclient.rpush "pending-updates-list", doc_key, callback diff --git a/services/real-time/test/acceptance/coffee/ApplyUpdateTests.coffee b/services/real-time/test/acceptance/coffee/ApplyUpdateTests.coffee index bb2df04ae2..0acc41cb70 100644 --- a/services/real-time/test/acceptance/coffee/ApplyUpdateTests.coffee +++ b/services/real-time/test/acceptance/coffee/ApplyUpdateTests.coffee @@ -46,11 +46,6 @@ describe "applyOtUpdate", -> rclient.lrange "pending-updates-list", 0, -1, (error, [doc_id]) => doc_id.should.equal "#{@project_id}:#{@doc_id}" done() - - it "should add the doc to the pending updates set in redis", (done) -> - rclient.sismember "DocsWithPendingUpdates", "#{@project_id}:#{@doc_id}", (error, isMember) => - isMember.should.equal 1 - done() it "should push the update into redis", (done) -> rclient.lrange "PendingUpdates:#{@doc_id}", 0, -1, (error, [update]) => @@ -145,11 +140,6 @@ describe "applyOtUpdate", -> rclient.lrange "pending-updates-list", 0, -1, (error, [doc_id]) => doc_id.should.equal "#{@project_id}:#{@doc_id}" done() - - it "should add the doc to the pending updates set in redis", (done) -> - rclient.sismember "DocsWithPendingUpdates", "#{@project_id}:#{@doc_id}", (error, isMember) => - isMember.should.equal 1 - done() it "should push the update into redis", (done) -> rclient.lrange "PendingUpdates:#{@doc_id}", 0, -1, (error, [update]) => diff --git a/services/real-time/test/unit/coffee/DocumentUpdaterManagerTests.coffee b/services/real-time/test/unit/coffee/DocumentUpdaterManagerTests.coffee index c693880e39..e6f9b2098b 100644 --- a/services/real-time/test/unit/coffee/DocumentUpdaterManagerTests.coffee +++ b/services/real-time/test/unit/coffee/DocumentUpdaterManagerTests.coffee @@ -123,10 +123,7 @@ describe 'DocumentUpdaterManager', -> "range":{"start":{"row":2,"column":2},"end":{"row":2,"column":3}}, "text":"e" } - @rclient.multi = sinon.stub().returns @rclient - @rclient.exec = sinon.stub().callsArg(0) - @rclient.rpush = sinon.stub() - @rclient.sadd = sinon.stub() + @rclient.rpush = sinon.stub().yields() @callback = sinon.stub() describe "successfully", -> @@ -143,14 +140,9 @@ describe 'DocumentUpdaterManager', -> .calledWith("pending-updates-list", "#{@project_id}:#{@doc_id}") .should.equal true - it "should push the doc id into the pending updates set", -> - @rclient.sadd - .calledWith("DocsWithPendingUpdates", "#{@project_id}:#{@doc_id}") - .should.equal true - - describe "with error connecting to redis during exec", -> + describe "with error talking to redis during rpush", -> beforeEach -> - @rclient.exec = sinon.stub().callsArgWith(0, new Error("something went wrong")) + @rclient.rpush = sinon.stub().yields(new Error("something went wrong")) @DocumentUpdaterManager.queueChange(@project_id, @doc_id, @change, @callback) it "should return an error", -> From 642134da796fe61ff2a36308a8f8f0801448e970 Mon Sep 17 00:00:00 2001 From: James Allen Date: Tue, 16 May 2017 11:06:05 +0100 Subject: [PATCH 084/491] Don't return a user if there is no entry ioredis returns a blank object, {}, if there is no key with hgetall. Previously, node-redis returned nil. So we need to check for a blank object as well as a nil object. --- .../app/coffee/ConnectedUsersManager.coffee | 2 +- .../unit/coffee/ConnectedUsersManagerTests.coffee | 13 ++++++++++--- 2 files changed, 11 insertions(+), 4 deletions(-) diff --git a/services/real-time/app/coffee/ConnectedUsersManager.coffee b/services/real-time/app/coffee/ConnectedUsersManager.coffee index 46454471ba..831a237356 100644 --- a/services/real-time/app/coffee/ConnectedUsersManager.coffee +++ b/services/real-time/app/coffee/ConnectedUsersManager.coffee @@ -50,7 +50,7 @@ module.exports = _getConnectedUser: (project_id, client_id, callback)-> rclient.hgetall Keys.connectedUser({project_id, client_id}), (err, result)-> - if !result? + if !result? or Object.keys(result).length == 0 result = connected : false client_id:client_id diff --git a/services/real-time/test/unit/coffee/ConnectedUsersManagerTests.coffee b/services/real-time/test/unit/coffee/ConnectedUsersManagerTests.coffee index 9f77cb0e8b..7d83cf1a0d 100644 --- a/services/real-time/test/unit/coffee/ConnectedUsersManagerTests.coffee +++ b/services/real-time/test/unit/coffee/ConnectedUsersManagerTests.coffee @@ -122,7 +122,7 @@ describe "ConnectedUsersManager", -> describe "_getConnectedUser", -> - it "should get the user returning connected if there is a value", (done)-> + it "should return a connected user if there is a user object", (done)-> cursorData = JSON.stringify(cursorData:{row:1}) @rClient.hgetall.callsArgWith(1, null, {connected_at:new Date(), cursorData}) @ConnectedUsersManager._getConnectedUser @project_id, @client_id, (err, result)=> @@ -130,8 +130,15 @@ describe "ConnectedUsersManager", -> result.client_id.should.equal @client_id done() - it "should get the user returning connected if there is a value", (done)-> - @rClient.hgetall.callsArgWith(1) + it "should return a not connected user if there is no object", (done)-> + @rClient.hgetall.callsArgWith(1, null, null) + @ConnectedUsersManager._getConnectedUser @project_id, @client_id, (err, result)=> + result.connected.should.equal false + result.client_id.should.equal @client_id + done() + + it "should return a not connected user if there is an empty object", (done)-> + @rClient.hgetall.callsArgWith(1, null, {}) @ConnectedUsersManager._getConnectedUser @project_id, @client_id, (err, result)=> result.connected.should.equal false result.client_id.should.equal @client_id From ab6fe1d948450013d067399465f59c82379dabbc Mon Sep 17 00:00:00 2001 From: Brian Gough Date: Thu, 1 Jun 2017 11:27:56 +0100 Subject: [PATCH 085/491] check for null bytes from JSON.stringify --- .../app/coffee/DocumentUpdaterManager.coffee | 4 +++ .../coffee/DocumentUpdaterManagerTests.coffee | 30 ++++++++++++++----- 2 files changed, 26 insertions(+), 8 deletions(-) diff --git a/services/real-time/app/coffee/DocumentUpdaterManager.coffee b/services/real-time/app/coffee/DocumentUpdaterManager.coffee index ab893aa47f..e183d11cec 100644 --- a/services/real-time/app/coffee/DocumentUpdaterManager.coffee +++ b/services/real-time/app/coffee/DocumentUpdaterManager.coffee @@ -54,6 +54,10 @@ module.exports = DocumentUpdaterManager = queueChange: (project_id, doc_id, change, callback = ()->)-> jsonChange = JSON.stringify change + if jsonChange.indexOf("\u0000") != -1 + error = new Error("null bytes found in op") + logger.error err: error, project_id: project_id, doc_id: doc_id, jsonChange: jsonChange, error.message + return callback(error) doc_key = "#{project_id}:#{doc_id}" # Push onto pendingUpdates for doc_id first, because once the doc updater # gets an entry on pending-updates-list, it starts processing. diff --git a/services/real-time/test/unit/coffee/DocumentUpdaterManagerTests.coffee b/services/real-time/test/unit/coffee/DocumentUpdaterManagerTests.coffee index e6f9b2098b..2d3e0ad3af 100644 --- a/services/real-time/test/unit/coffee/DocumentUpdaterManagerTests.coffee +++ b/services/real-time/test/unit/coffee/DocumentUpdaterManagerTests.coffee @@ -17,14 +17,17 @@ describe 'DocumentUpdaterManager', -> pendingUpdates: ({doc_id}) -> "PendingUpdates:#{doc_id}" @rclient = {auth:->} - @DocumentUpdaterManager = SandboxedModule.require modulePath, requires: - 'settings-sharelatex':@settings - 'logger-sharelatex': @logger = {log: sinon.stub(), error: sinon.stub(), warn: sinon.stub()} - 'request': @request = {} - 'redis-sharelatex' : createClient: () => @rclient - 'metrics-sharelatex': @Metrics = - Timer: class Timer - done: () -> + @DocumentUpdaterManager = SandboxedModule.require modulePath, + requires: + 'settings-sharelatex':@settings + 'logger-sharelatex': @logger = {log: sinon.stub(), error: sinon.stub(), warn: sinon.stub()} + 'request': @request = {} + 'redis-sharelatex' : createClient: () => @rclient + 'metrics-sharelatex': @Metrics = + Timer: class Timer + done: () -> + globals: + JSON: @JSON = Object.create(JSON) # avoid modifying JSON object directly describe "getDocument", -> beforeEach -> @@ -147,3 +150,14 @@ describe 'DocumentUpdaterManager', -> it "should return an error", -> @callback.calledWithExactly(sinon.match(Error)).should.equal true + + describe "with null byte corruption", -> + beforeEach -> + @JSON.stringify = () -> return '["bad bytes! \u0000 <- here"]' + @DocumentUpdaterManager.queueChange(@project_id, @doc_id, @change, @callback) + + it "should return an error", -> + @callback.calledWithExactly(sinon.match(Error)).should.equal true + + it "should not push the change onto the pending-updates-list queue", -> + @rclient.rpush.called.should.equal false From 3dae68a4f0214bc6180f36f06781a303186fc7eb Mon Sep 17 00:00:00 2001 From: Joe Green Date: Fri, 11 Aug 2017 14:24:46 +0100 Subject: [PATCH 086/491] Create Jenkinsfile --- services/real-time/Jenkinsfile | 77 ++++++++++++++++++++++++++++++++++ 1 file changed, 77 insertions(+) create mode 100644 services/real-time/Jenkinsfile diff --git a/services/real-time/Jenkinsfile b/services/real-time/Jenkinsfile new file mode 100644 index 0000000000..7a0be11b68 --- /dev/null +++ b/services/real-time/Jenkinsfile @@ -0,0 +1,77 @@ +pipeline { + + agent { + docker { + image 'node:4.2.1' + args "-v /var/lib/jenkins/.npm:/tmp/.npm" + } + } + + environment { + HOME = "/tmp" + } + + triggers { + pollSCM('* * * * *') + cron('@daily') + } + + stages { + stage('Set up') { + steps { + // we need to disable logallrefupdates, else git clones during the npm install will require git to lookup the user id + // which does not exist in the container's /etc/passwd file, causing the clone to fail. + sh 'git config --global core.logallrefupdates false' + } + } + stage('Install') { + steps { + sh 'rm -fr node_modules' + sh 'npm install' + sh 'npm rebuild' + sh 'npm install --quiet grunt-cli' + } + } + stage('Compile') { + steps { + sh 'node_modules/.bin/grunt install' + } + } + stage('Test') { + steps { + sh 'node_modules/.bin/grunt test:unit' + } + } + stage('Package') { + steps { + sh 'touch build.tar.gz' // Avoid tar warning about files changing during read + sh 'tar -czf build.tar.gz --exclude=build.tar.gz --exclude-vcs .' + } + } + stage('Publish') { + steps { + withAWS(credentials:'S3_CI_BUILDS_AWS_KEYS', region:"${S3_REGION_BUILD_ARTEFACTS}") { + s3Upload(file:'build.tar.gz', bucket:"${S3_BUCKET_BUILD_ARTEFACTS}", path:"${JOB_NAME}/${BUILD_NUMBER}.tar.gz") + } + } + } + } + + post { + failure { + mail(from: "${EMAIL_ALERT_FROM}", + to: "${EMAIL_ALERT_TO}", + subject: "Jenkins build failed: ${JOB_NAME}:${BUILD_NUMBER}", + body: "Build: ${BUILD_URL}") + } + } + + // The options directive is for configuration that applies to the whole job. + options { + // we'd like to make sure remove old builds, so we don't fill up our storage! + buildDiscarder(logRotator(numToKeepStr:'50')) + + // And we'd really like to be sure that this build doesn't hang forever, so let's time it out after: + timeout(time: 30, unit: 'MINUTES') + } +} From 72ef354206711546144d2efe2538e19ef420cedf Mon Sep 17 00:00:00 2001 From: Joe Green Date: Mon, 4 Sep 2017 14:54:05 +0100 Subject: [PATCH 087/491] build.txt --- services/real-time/Jenkinsfile | 3 +++ 1 file changed, 3 insertions(+) diff --git a/services/real-time/Jenkinsfile b/services/real-time/Jenkinsfile index 7a0be11b68..81456dee00 100644 --- a/services/real-time/Jenkinsfile +++ b/services/real-time/Jenkinsfile @@ -44,6 +44,7 @@ pipeline { } stage('Package') { steps { + sh 'echo ${BUILD_NUMBER} > build_number.txt' sh 'touch build.tar.gz' // Avoid tar warning about files changing during read sh 'tar -czf build.tar.gz --exclude=build.tar.gz --exclude-vcs .' } @@ -52,6 +53,8 @@ pipeline { steps { withAWS(credentials:'S3_CI_BUILDS_AWS_KEYS', region:"${S3_REGION_BUILD_ARTEFACTS}") { s3Upload(file:'build.tar.gz', bucket:"${S3_BUCKET_BUILD_ARTEFACTS}", path:"${JOB_NAME}/${BUILD_NUMBER}.tar.gz") + // The deployment process uses this file to figure out the latest build + s3Upload(file:'build_number.txt', bucket:"${S3_BUCKET_BUILD_ARTEFACTS}", path:"${JOB_NAME}/latest") } } } From 0b41d32cbbf783bb58bcfeddcd17507c9336c16d Mon Sep 17 00:00:00 2001 From: Joe Green Date: Thu, 21 Sep 2017 10:59:41 +0100 Subject: [PATCH 088/491] Update Jenkinsfile --- services/real-time/Jenkinsfile | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/services/real-time/Jenkinsfile b/services/real-time/Jenkinsfile index 81456dee00..f5f121ecf3 100644 --- a/services/real-time/Jenkinsfile +++ b/services/real-time/Jenkinsfile @@ -35,6 +35,7 @@ pipeline { stage('Compile') { steps { sh 'node_modules/.bin/grunt install' + sh 'node_modules/.bin/grunt compile:acceptance_tests' } } stage('Test') { @@ -42,6 +43,12 @@ pipeline { sh 'node_modules/.bin/grunt test:unit' } } + stage('Acceptance Tests') { + steps { + sh 'docker pull sharelatex/acceptance-test-runner' + sh 'docker run --rm -v $(pwd):/app sharelatex/acceptance-test-runner' + } + } stage('Package') { steps { sh 'echo ${BUILD_NUMBER} > build_number.txt' From dc8f4ffc2a48ef2309f44af2a788dd2c1468d4c2 Mon Sep 17 00:00:00 2001 From: Joe Green Date: Thu, 21 Sep 2017 11:03:07 +0100 Subject: [PATCH 089/491] Update Jenkinsfile --- services/real-time/Jenkinsfile | 33 +++++++++++++++++---------------- 1 file changed, 17 insertions(+), 16 deletions(-) diff --git a/services/real-time/Jenkinsfile b/services/real-time/Jenkinsfile index f5f121ecf3..b684c49eb0 100644 --- a/services/real-time/Jenkinsfile +++ b/services/real-time/Jenkinsfile @@ -1,11 +1,6 @@ pipeline { - agent { - docker { - image 'node:4.2.1' - args "-v /var/lib/jenkins/.npm:/tmp/.npm" - } - } + agent any environment { HOME = "/tmp" @@ -17,29 +12,35 @@ pipeline { } stages { - stage('Set up') { + stage('Install') { + agent { + docker { + image 'node:4.2.1' + args "-v /var/lib/jenkins/.npm:/tmp/.npm -e HOME=/tmp" + reuseNode true + } + } steps { // we need to disable logallrefupdates, else git clones during the npm install will require git to lookup the user id // which does not exist in the container's /etc/passwd file, causing the clone to fail. sh 'git config --global core.logallrefupdates false' - } - } - stage('Install') { - steps { sh 'rm -fr node_modules' sh 'npm install' sh 'npm rebuild' sh 'npm install --quiet grunt-cli' } } - stage('Compile') { + stage('Compile and Test') { + agent { + docker { + image 'node:4.2.1' + args "-v /var/lib/jenkins/.npm:/tmp/.npm -e HOME=/tmp" + reuseNode true + } + } steps { sh 'node_modules/.bin/grunt install' sh 'node_modules/.bin/grunt compile:acceptance_tests' - } - } - stage('Test') { - steps { sh 'node_modules/.bin/grunt test:unit' } } From 9f503f1e9f857d90a06d53122c2ba20cb6388375 Mon Sep 17 00:00:00 2001 From: Alasdair Smith Date: Thu, 21 Sep 2017 13:25:55 +0100 Subject: [PATCH 090/491] First pass at encoding changes & comments in ranges --- .../app/coffee/WebsocketController.coffee | 24 +++++++++++++++++++ 1 file changed, 24 insertions(+) diff --git a/services/real-time/app/coffee/WebsocketController.coffee b/services/real-time/app/coffee/WebsocketController.coffee index e0242d4208..e4057baf7c 100644 --- a/services/real-time/app/coffee/WebsocketController.coffee +++ b/services/real-time/app/coffee/WebsocketController.coffee @@ -101,6 +101,30 @@ module.exports = WebsocketController = logger.err {err, project_id, doc_id, fromVersion, line, client_id: client.id}, "error encoding line uri component" return callback(err) escapedLines.push line + + if ranges.comments + escapedComments = [] + for comment in ranges.comments + try + comment.op.c = unescape(encodeURIComponent(comment.op.c)) + catch err + logger.err {err, project_id, doc_id, fromVersion, comment, client_id: client.id}, "error encoding comment uri component" + return callback(err) + escapedComments.push comment + ranges.comments = escapedComments + + if ranges.changes + escapedChanges = [] + for change in ranges.changes + try + change.op.i = unescape(encodeURIComponent(change.op.i)) if change.op.i + change.op.d = unescape(encodeURIComponent(change.op.d)) if change.op.d + catch err + logger.err {err, project_id, doc_id, fromVersion, change, client_id: client.id}, "error encoding change uri component" + return callback(err) + escapedChanges.push change + ranges.changes = escapedChanges + AuthorizationManager.addAccessToDoc client, doc_id client.join(doc_id) callback null, escapedLines, version, ops, ranges From aa6e0d0d6977c5a25998b3258943eee1ea20540b Mon Sep 17 00:00:00 2001 From: Alasdair Smith Date: Thu, 21 Sep 2017 14:23:16 +0100 Subject: [PATCH 091/491] Only encode ranges if option passed --- services/real-time/app/coffee/Router.coffee | 4 +- .../app/coffee/WebsocketController.coffee | 47 +++++++++---------- 2 files changed, 25 insertions(+), 26 deletions(-) diff --git a/services/real-time/app/coffee/Router.coffee b/services/real-time/app/coffee/Router.coffee index 2cc655eafc..8821748a35 100644 --- a/services/real-time/app/coffee/Router.coffee +++ b/services/real-time/app/coffee/Router.coffee @@ -80,13 +80,13 @@ module.exports = Router = Router._handleError null, err, client, "leaveProject" - client.on "joinDoc", (doc_id, fromVersion, callback) -> + client.on "joinDoc", (doc_id, options, fromVersion, callback) -> # fromVersion is optional if typeof fromVersion == "function" callback = fromVersion fromVersion = -1 - WebsocketController.joinDoc client, doc_id, fromVersion, (err, args...) -> + WebsocketController.joinDoc client, doc_id, options, fromVersion, (err, args...) -> if err? Router._handleError callback, err, client, "joinDoc", {doc_id, fromVersion} else diff --git a/services/real-time/app/coffee/WebsocketController.coffee b/services/real-time/app/coffee/WebsocketController.coffee index e4057baf7c..dec4d3d142 100644 --- a/services/real-time/app/coffee/WebsocketController.coffee +++ b/services/real-time/app/coffee/WebsocketController.coffee @@ -80,7 +80,7 @@ module.exports = WebsocketController = callback() , WebsocketController.FLUSH_IF_EMPTY_DELAY - joinDoc: (client, doc_id, fromVersion = -1, callback = (error, doclines, version, ops, ranges) ->) -> + joinDoc: (client, doc_id, options, fromVersion = -1, callback = (error, doclines, version, ops, ranges) ->) -> metrics.inc "editor.join-doc" Utils.getClientAttributes client, ["project_id", "user_id"], (error, {project_id, user_id}) -> return callback(error) if error? @@ -101,29 +101,28 @@ module.exports = WebsocketController = logger.err {err, project_id, doc_id, fromVersion, line, client_id: client.id}, "error encoding line uri component" return callback(err) escapedLines.push line - - if ranges.comments - escapedComments = [] - for comment in ranges.comments - try - comment.op.c = unescape(encodeURIComponent(comment.op.c)) - catch err - logger.err {err, project_id, doc_id, fromVersion, comment, client_id: client.id}, "error encoding comment uri component" - return callback(err) - escapedComments.push comment - ranges.comments = escapedComments - - if ranges.changes - escapedChanges = [] - for change in ranges.changes - try - change.op.i = unescape(encodeURIComponent(change.op.i)) if change.op.i - change.op.d = unescape(encodeURIComponent(change.op.d)) if change.op.d - catch err - logger.err {err, project_id, doc_id, fromVersion, change, client_id: client.id}, "error encoding change uri component" - return callback(err) - escapedChanges.push change - ranges.changes = escapedChanges + if options.encodeRanges + if ranges.comments + escapedComments = [] + for comment in ranges.comments + try + comment.op.c = unescape(encodeURIComponent(comment.op.c)) + catch err + logger.err {err, project_id, doc_id, fromVersion, comment, client_id: client.id}, "error encoding comment uri component" + return callback(err) + escapedComments.push comment + ranges.comments = escapedComments + if ranges.changes + escapedChanges = [] + for change in ranges.changes + try + change.op.i = unescape(encodeURIComponent(change.op.i)) if change.op.i + change.op.d = unescape(encodeURIComponent(change.op.d)) if change.op.d + catch err + logger.err {err, project_id, doc_id, fromVersion, change, client_id: client.id}, "error encoding change uri component" + return callback(err) + escapedChanges.push change + ranges.changes = escapedChanges AuthorizationManager.addAccessToDoc client, doc_id client.join(doc_id) From b796879c9f6d4f262349a1c267c9453608d06e26 Mon Sep 17 00:00:00 2001 From: Alasdair Smith Date: Thu, 21 Sep 2017 14:58:49 +0100 Subject: [PATCH 092/491] Handle options not being passed --- services/real-time/app/coffee/Router.coffee | 16 +++++++++++++++- 1 file changed, 15 insertions(+), 1 deletion(-) diff --git a/services/real-time/app/coffee/Router.coffee b/services/real-time/app/coffee/Router.coffee index 8821748a35..e6dda566e0 100644 --- a/services/real-time/app/coffee/Router.coffee +++ b/services/real-time/app/coffee/Router.coffee @@ -79,8 +79,22 @@ module.exports = Router = if err? Router._handleError null, err, client, "leaveProject" - + # Variadic. The possible options: + # doc_id, callback + # doc_id, fromVersion, callback + # doc_id, options, callback + # doc_id, options, fromVersion, callback client.on "joinDoc", (doc_id, options, fromVersion, callback) -> + # options is optional + if typeof options == "function" + options = {} + callback = options + fromVersion = -1 + else if typeof options == "number" + options = {} + fromVersion = options + callback = fromVersion + # fromVersion is optional if typeof fromVersion == "function" callback = fromVersion From 55c880e1dd12a2bd777e38da1116d4d30c9104d7 Mon Sep 17 00:00:00 2001 From: Alasdair Smith Date: Thu, 21 Sep 2017 15:07:15 +0100 Subject: [PATCH 093/491] DRY up a bit --- .../app/coffee/WebsocketController.coffee | 34 +++++++------------ 1 file changed, 12 insertions(+), 22 deletions(-) diff --git a/services/real-time/app/coffee/WebsocketController.coffee b/services/real-time/app/coffee/WebsocketController.coffee index dec4d3d142..70d325197f 100644 --- a/services/real-time/app/coffee/WebsocketController.coffee +++ b/services/real-time/app/coffee/WebsocketController.coffee @@ -91,38 +91,28 @@ module.exports = WebsocketController = return callback(error) if error? DocumentUpdaterManager.getDocument project_id, doc_id, fromVersion, (error, lines, version, ranges, ops) -> return callback(error) if error? + # Encode any binary bits of data so it can go via WebSockets # See http://ecmanaut.blogspot.co.uk/2006/07/encoding-decoding-utf8-in-javascript.html + encodeForWebsockets = (text) -> unescape(encodeURIComponent(text)) escapedLines = [] for line in lines try - line = unescape(encodeURIComponent(line)) + line = encodeForWebsockets(line) catch err logger.err {err, project_id, doc_id, fromVersion, line, client_id: client.id}, "error encoding line uri component" return callback(err) escapedLines.push line if options.encodeRanges - if ranges.comments - escapedComments = [] - for comment in ranges.comments - try - comment.op.c = unescape(encodeURIComponent(comment.op.c)) - catch err - logger.err {err, project_id, doc_id, fromVersion, comment, client_id: client.id}, "error encoding comment uri component" - return callback(err) - escapedComments.push comment - ranges.comments = escapedComments - if ranges.changes - escapedChanges = [] - for change in ranges.changes - try - change.op.i = unescape(encodeURIComponent(change.op.i)) if change.op.i - change.op.d = unescape(encodeURIComponent(change.op.d)) if change.op.d - catch err - logger.err {err, project_id, doc_id, fromVersion, change, client_id: client.id}, "error encoding change uri component" - return callback(err) - escapedChanges.push change - ranges.changes = escapedChanges + try + for comment in ranges?.comments or [] + comment.op.c = encodeForWebsockets(comment.op.c) + for change in ranges?.changes or [] + change.op.i = encodeForWebsockets(comment.op.i) if change.op.i + change.op.d = encodeForWebsockets(comment.op.d) if change.op.d + catch err + logger.err {err, project_id, doc_id, fromVersion, ranges, client_id: client.id}, "error encoding range uri component" + return callback(err) AuthorizationManager.addAccessToDoc client, doc_id client.join(doc_id) From 790b9ea8edb17c92a49ccdd15c7f818885c672a1 Mon Sep 17 00:00:00 2001 From: Alasdair Smith Date: Thu, 21 Sep 2017 15:19:19 +0100 Subject: [PATCH 094/491] Switch order of args --- services/real-time/app/coffee/Router.coffee | 25 +++++++++------------ 1 file changed, 11 insertions(+), 14 deletions(-) diff --git a/services/real-time/app/coffee/Router.coffee b/services/real-time/app/coffee/Router.coffee index e6dda566e0..a55147bfb2 100644 --- a/services/real-time/app/coffee/Router.coffee +++ b/services/real-time/app/coffee/Router.coffee @@ -79,26 +79,23 @@ module.exports = Router = if err? Router._handleError null, err, client, "leaveProject" - # Variadic. The possible options: + # Variadic. The possible arguments: # doc_id, callback # doc_id, fromVersion, callback # doc_id, options, callback - # doc_id, options, fromVersion, callback - client.on "joinDoc", (doc_id, options, fromVersion, callback) -> - # options is optional + # doc_id, fromVersion, options, callback + client.on "joinDoc", (doc_id, fromVersion, options, callback) -> + if typeof fromVersion == "function" + fromVersion = -1 + options = {} + callback = fromVersion + else if typeof fromVersion == "object" + fromVersion = -1 + options = fromVersion + callback = options if typeof options == "function" options = {} callback = options - fromVersion = -1 - else if typeof options == "number" - options = {} - fromVersion = options - callback = fromVersion - - # fromVersion is optional - if typeof fromVersion == "function" - callback = fromVersion - fromVersion = -1 WebsocketController.joinDoc client, doc_id, options, fromVersion, (err, args...) -> if err? From 3966e2f85bcfd3fcbf742d331af55eb3c2a9b939 Mon Sep 17 00:00:00 2001 From: Alasdair Smith Date: Thu, 21 Sep 2017 16:55:49 +0100 Subject: [PATCH 095/491] Make variadic options more explicit --- services/real-time/app/coffee/Router.coffee | 19 ++++++++++++------- 1 file changed, 12 insertions(+), 7 deletions(-) diff --git a/services/real-time/app/coffee/Router.coffee b/services/real-time/app/coffee/Router.coffee index a55147bfb2..b3d6ffeb16 100644 --- a/services/real-time/app/coffee/Router.coffee +++ b/services/real-time/app/coffee/Router.coffee @@ -85,17 +85,22 @@ module.exports = Router = # doc_id, options, callback # doc_id, fromVersion, options, callback client.on "joinDoc", (doc_id, fromVersion, options, callback) -> - if typeof fromVersion == "function" - fromVersion = -1 - options = {} + if typeof fromVersion == "function" and !options callback = fromVersion - else if typeof fromVersion == "object" fromVersion = -1 - options = fromVersion - callback = options - if typeof options == "function" options = {} + else if typeof fromVersion == "number" and typeof options == "function" callback = options + options = {} + else if typeof fromVersion == "object" and typeof options == "function" + callback = options + options = fromVersion + fromVersion = -1 + else if typeof fromVersion == "number" and typeof options == "object" + # Called with 4 args, things are as expected + else + logger.error { arguments: arguments }, "unexpected arguments" + return {} # ???? WebsocketController.joinDoc client, doc_id, options, fromVersion, (err, args...) -> if err? From 90d05dc6ddeb93dbb44405880874de1e3bb6470d Mon Sep 17 00:00:00 2001 From: Alasdair Smith Date: Thu, 21 Sep 2017 16:56:09 +0100 Subject: [PATCH 096/491] Make args order consistent --- services/real-time/app/coffee/Router.coffee | 2 +- services/real-time/app/coffee/WebsocketController.coffee | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/services/real-time/app/coffee/Router.coffee b/services/real-time/app/coffee/Router.coffee index b3d6ffeb16..0fb502bd4f 100644 --- a/services/real-time/app/coffee/Router.coffee +++ b/services/real-time/app/coffee/Router.coffee @@ -102,7 +102,7 @@ module.exports = Router = logger.error { arguments: arguments }, "unexpected arguments" return {} # ???? - WebsocketController.joinDoc client, doc_id, options, fromVersion, (err, args...) -> + WebsocketController.joinDoc client, doc_id, fromVersion, options, (err, args...) -> if err? Router._handleError callback, err, client, "joinDoc", {doc_id, fromVersion} else diff --git a/services/real-time/app/coffee/WebsocketController.coffee b/services/real-time/app/coffee/WebsocketController.coffee index 70d325197f..4b51da8d53 100644 --- a/services/real-time/app/coffee/WebsocketController.coffee +++ b/services/real-time/app/coffee/WebsocketController.coffee @@ -80,7 +80,7 @@ module.exports = WebsocketController = callback() , WebsocketController.FLUSH_IF_EMPTY_DELAY - joinDoc: (client, doc_id, options, fromVersion = -1, callback = (error, doclines, version, ops, ranges) ->) -> + joinDoc: (client, doc_id, fromVersion = -1, options, callback = (error, doclines, version, ops, ranges) ->) -> metrics.inc "editor.join-doc" Utils.getClientAttributes client, ["project_id", "user_id"], (error, {project_id, user_id}) -> return callback(error) if error? From a299d7335d847cec2a85a32fe07b43dded07a894 Mon Sep 17 00:00:00 2001 From: Alasdair Smith Date: Thu, 21 Sep 2017 16:56:18 +0100 Subject: [PATCH 097/491] Fix incorrect var --- services/real-time/app/coffee/WebsocketController.coffee | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/services/real-time/app/coffee/WebsocketController.coffee b/services/real-time/app/coffee/WebsocketController.coffee index 4b51da8d53..580e8ac20b 100644 --- a/services/real-time/app/coffee/WebsocketController.coffee +++ b/services/real-time/app/coffee/WebsocketController.coffee @@ -108,8 +108,8 @@ module.exports = WebsocketController = for comment in ranges?.comments or [] comment.op.c = encodeForWebsockets(comment.op.c) for change in ranges?.changes or [] - change.op.i = encodeForWebsockets(comment.op.i) if change.op.i - change.op.d = encodeForWebsockets(comment.op.d) if change.op.d + change.op.i = encodeForWebsockets(change.op.i) if change.op.i + change.op.d = encodeForWebsockets(change.op.d) if change.op.d catch err logger.err {err, project_id, doc_id, fromVersion, ranges, client_id: client.id}, "error encoding range uri component" return callback(err) From 5d8e201732eeaff7da2ffa6e8fec36804f3d2742 Mon Sep 17 00:00:00 2001 From: Alasdair Smith Date: Thu, 21 Sep 2017 16:58:03 +0100 Subject: [PATCH 098/491] Don't return obj --- services/real-time/app/coffee/Router.coffee | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/services/real-time/app/coffee/Router.coffee b/services/real-time/app/coffee/Router.coffee index 0fb502bd4f..21d9e34b7d 100644 --- a/services/real-time/app/coffee/Router.coffee +++ b/services/real-time/app/coffee/Router.coffee @@ -100,7 +100,7 @@ module.exports = Router = # Called with 4 args, things are as expected else logger.error { arguments: arguments }, "unexpected arguments" - return {} # ???? + return # ???? WebsocketController.joinDoc client, doc_id, fromVersion, options, (err, args...) -> if err? From 937bf82a2fee9fa166f45df5f210cc0866e75f58 Mon Sep 17 00:00:00 2001 From: Alasdair Smith Date: Fri, 22 Sep 2017 09:25:24 +0100 Subject: [PATCH 099/491] Return callback with guard --- services/real-time/app/coffee/Router.coffee | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/services/real-time/app/coffee/Router.coffee b/services/real-time/app/coffee/Router.coffee index 21d9e34b7d..17107e443a 100644 --- a/services/real-time/app/coffee/Router.coffee +++ b/services/real-time/app/coffee/Router.coffee @@ -100,7 +100,7 @@ module.exports = Router = # Called with 4 args, things are as expected else logger.error { arguments: arguments }, "unexpected arguments" - return # ???? + return callback?(new Error("unexpected arguments")) WebsocketController.joinDoc client, doc_id, fromVersion, options, (err, args...) -> if err? From c67150ea10334f5ab57445a6f1b31b852d5750cf Mon Sep 17 00:00:00 2001 From: Alasdair Smith Date: Fri, 22 Sep 2017 09:33:29 +0100 Subject: [PATCH 100/491] Ensure falsy value doesn't fail conditional --- services/real-time/app/coffee/WebsocketController.coffee | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/services/real-time/app/coffee/WebsocketController.coffee b/services/real-time/app/coffee/WebsocketController.coffee index 580e8ac20b..63c3ff903a 100644 --- a/services/real-time/app/coffee/WebsocketController.coffee +++ b/services/real-time/app/coffee/WebsocketController.coffee @@ -108,8 +108,8 @@ module.exports = WebsocketController = for comment in ranges?.comments or [] comment.op.c = encodeForWebsockets(comment.op.c) for change in ranges?.changes or [] - change.op.i = encodeForWebsockets(change.op.i) if change.op.i - change.op.d = encodeForWebsockets(change.op.d) if change.op.d + change.op.i = encodeForWebsockets(change.op.i) if change.op.i? + change.op.d = encodeForWebsockets(change.op.d) if change.op.d? catch err logger.err {err, project_id, doc_id, fromVersion, ranges, client_id: client.id}, "error encoding range uri component" return callback(err) From a0505afb23a63b39bf81ed7b4fa27ccaef3996d1 Mon Sep 17 00:00:00 2001 From: Alasdair Smith Date: Fri, 22 Sep 2017 09:34:10 +0100 Subject: [PATCH 101/491] Be defensive on comment text --- services/real-time/app/coffee/WebsocketController.coffee | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/services/real-time/app/coffee/WebsocketController.coffee b/services/real-time/app/coffee/WebsocketController.coffee index 63c3ff903a..53ccecd51b 100644 --- a/services/real-time/app/coffee/WebsocketController.coffee +++ b/services/real-time/app/coffee/WebsocketController.coffee @@ -106,7 +106,7 @@ module.exports = WebsocketController = if options.encodeRanges try for comment in ranges?.comments or [] - comment.op.c = encodeForWebsockets(comment.op.c) + comment.op.c = encodeForWebsockets(comment.op.c) if comment.op.c? for change in ranges?.changes or [] change.op.i = encodeForWebsockets(change.op.i) if change.op.i? change.op.d = encodeForWebsockets(change.op.d) if change.op.d? From b8d3f34e5463f9c4dda0942ff9221c62c62d3f9f Mon Sep 17 00:00:00 2001 From: Alasdair Smith Date: Fri, 22 Sep 2017 10:42:51 +0100 Subject: [PATCH 102/491] Fix joinDoc tests not passing options arg and restructure tests --- .../coffee/WebsocketControllerTests.coffee | 38 ++++++++++++------- 1 file changed, 24 insertions(+), 14 deletions(-) diff --git a/services/real-time/test/unit/coffee/WebsocketControllerTests.coffee b/services/real-time/test/unit/coffee/WebsocketControllerTests.coffee index 8db81b716e..f0ae54ae61 100644 --- a/services/real-time/test/unit/coffee/WebsocketControllerTests.coffee +++ b/services/real-time/test/unit/coffee/WebsocketControllerTests.coffee @@ -221,49 +221,59 @@ describe 'WebsocketController', -> @version = 42 @ops = ["mock", "ops"] @ranges = { "mock": "ranges" } + @options = {} @client.params.project_id = @project_id @AuthorizationManager.addAccessToDoc = sinon.stub() @AuthorizationManager.assertClientCanViewProject = sinon.stub().callsArgWith(1, null) @DocumentUpdaterManager.getDocument = sinon.stub().callsArgWith(3, null, @doc_lines, @version, @ranges, @ops) - - describe "with a fromVersion", -> + + describe "works", -> beforeEach -> - @fromVersion = 40 - @WebsocketController.joinDoc @client, @doc_id, @fromVersion, @callback - + @WebsocketController.joinDoc @client, @doc_id, -1, @options, @callback + it "should check that the client is authorized to view the project", -> @AuthorizationManager.assertClientCanViewProject .calledWith(@client) .should.equal true - - it "should get the document from the DocumentUpdaterManager", -> + + it "should get the document from the DocumentUpdaterManager with fromVersion", -> @DocumentUpdaterManager.getDocument - .calledWith(@project_id, @doc_id, @fromVersion) + .calledWith(@project_id, @doc_id, -1) .should.equal true it "should add permissions for the client to access the doc", -> @AuthorizationManager.addAccessToDoc .calledWith(@client, @doc_id) .should.equal true - + it "should join the client to room for the doc_id", -> @client.join .calledWith(@doc_id) .should.equal true - + it "should call the callback with the lines, version, ranges and ops", -> @callback .calledWith(null, @doc_lines, @version, @ops, @ranges) .should.equal true - + it "should increment the join-doc metric", -> @metrics.inc.calledWith("editor.join-doc").should.equal true + + describe "with a fromVersion", -> + beforeEach -> + @fromVersion = 40 + @WebsocketController.joinDoc @client, @doc_id, @fromVersion, @options, @callback + it "should get the document from the DocumentUpdaterManager with fromVersion", -> + @DocumentUpdaterManager.getDocument + .calledWith(@project_id, @doc_id, @fromVersion) + .should.equal true + describe "with doclines that need escaping", -> beforeEach -> @doc_lines.push ["räksmörgås"] - @WebsocketController.joinDoc @client, @doc_id, -1, @callback + @WebsocketController.joinDoc @client, @doc_id, -1, @options, @callback it "should call the callback with the escaped lines", -> escaped_lines = @callback.args[0][1] @@ -271,11 +281,11 @@ describe 'WebsocketController', -> escaped_word.should.equal 'räksmörgÃ¥s' # Check that unescaping works decodeURIComponent(escape(escaped_word)).should.equal "räksmörgås" - + describe "when not authorized", -> beforeEach -> @AuthorizationManager.assertClientCanViewProject = sinon.stub().callsArgWith(1, @err = new Error("not authorized")) - @WebsocketController.joinDoc @client, @doc_id, -1, @callback + @WebsocketController.joinDoc @client, @doc_id, -1, @options, @callback it "should call the callback with an error", -> @callback.calledWith(@err).should.equal true From 785d6e2eeaa2e6642d19ba9acefaa8e00a59987f Mon Sep 17 00:00:00 2001 From: Alasdair Smith Date: Fri, 22 Sep 2017 10:43:11 +0100 Subject: [PATCH 103/491] Add tests for comment & change encoding --- .../coffee/WebsocketControllerTests.coffee | 30 +++++++++++++++++++ 1 file changed, 30 insertions(+) diff --git a/services/real-time/test/unit/coffee/WebsocketControllerTests.coffee b/services/real-time/test/unit/coffee/WebsocketControllerTests.coffee index f0ae54ae61..a8a3841faf 100644 --- a/services/real-time/test/unit/coffee/WebsocketControllerTests.coffee +++ b/services/real-time/test/unit/coffee/WebsocketControllerTests.coffee @@ -282,6 +282,36 @@ describe 'WebsocketController', -> # Check that unescaping works decodeURIComponent(escape(escaped_word)).should.equal "räksmörgås" + describe "with comments that need encoding", -> + beforeEach -> + @ranges.comments = [{ op: { c: "räksmörgås" } }] + @WebsocketController.joinDoc @client, @doc_id, -1, { encodeRanges: true }, @callback + + it "should call the callback with the encoded comment", -> + encoded_comments = @callback.args[0][4] + encoded_comment = encoded_comments.comments.pop() + encoded_comment_text = encoded_comment.op.c + encoded_comment_text.should.equal 'räksmörgÃ¥s' + + describe "with changes that need encoding", -> + it "should call the callback with the encoded insert change", -> + @ranges.changes = [{ op: { i: "räksmörgås" } }] + @WebsocketController.joinDoc @client, @doc_id, -1, { encodeRanges: true }, @callback + + encoded_changes = @callback.args[0][4] + encoded_change = encoded_changes.changes.pop() + encoded_change_text = encoded_change.op.i + encoded_change_text.should.equal 'räksmörgÃ¥s' + + it "should call the callback with the encoded delete change", -> + @ranges.changes = [{ op: { d: "räksmörgås" } }] + @WebsocketController.joinDoc @client, @doc_id, -1, { encodeRanges: true }, @callback + + encoded_changes = @callback.args[0][4] + encoded_change = encoded_changes.changes.pop() + encoded_change_text = encoded_change.op.d + encoded_change_text.should.equal 'räksmörgÃ¥s' + describe "when not authorized", -> beforeEach -> @AuthorizationManager.assertClientCanViewProject = sinon.stub().callsArgWith(1, @err = new Error("not authorized")) From 6470577c3fdb223e21c6573a8bd9bcbe83023a76 Mon Sep 17 00:00:00 2001 From: Alasdair Smith Date: Fri, 22 Sep 2017 11:01:11 +0100 Subject: [PATCH 104/491] Add acceptance tests for joinDoc variadic --- .../acceptance/coffee/JoinDocTests.coffee | 79 ++++++++++++++++++- 1 file changed, 78 insertions(+), 1 deletion(-) diff --git a/services/real-time/test/acceptance/coffee/JoinDocTests.coffee b/services/real-time/test/acceptance/coffee/JoinDocTests.coffee index a9d5406345..6c204b6079 100644 --- a/services/real-time/test/acceptance/coffee/JoinDocTests.coffee +++ b/services/real-time/test/acceptance/coffee/JoinDocTests.coffee @@ -166,4 +166,81 @@ describe "joinDoc", -> it "should have joined the doc room", (done) -> RealTimeClient.getConnectedClient @client.socket.sessionid, (error, client) => expect(@doc_id in client.rooms).to.equal true - done() \ No newline at end of file + done() + + describe "with options", -> + before (done) -> + @options = { encodeRanges: true } + async.series [ + (cb) => + FixturesManager.setUpProject { + privilegeLevel: "readAndWrite" + }, (e, {@project_id, @user_id}) => + cb(e) + + (cb) => + FixturesManager.setUpDoc @project_id, {@lines, @version, @ops, @ranges}, (e, {@doc_id}) => + cb(e) + + (cb) => + @client = RealTimeClient.connect() + @client.on "connectionAccepted", cb + + (cb) => + @client.emit "joinProject", project_id: @project_id, cb + + (cb) => + @client.emit "joinDoc", @doc_id, @options, (error, @returnedArgs...) => cb(error) + ], done + + it "should get the doc from the doc updater with the default fromVersion", -> + MockDocUpdaterServer.getDocument + .calledWith(@project_id, @doc_id, -1) + .should.equal true + + it "should return the doc lines, version, ranges and ops", -> + @returnedArgs.should.deep.equal [@lines, @version, @ops, @ranges] + + it "should have joined the doc room", (done) -> + RealTimeClient.getConnectedClient @client.socket.sessionid, (error, client) => + expect(@doc_id in client.rooms).to.equal true + done() + + describe "with fromVersion and options", -> + before (done) -> + @fromVersion = 36 + @options = { encodeRanges: true } + async.series [ + (cb) => + FixturesManager.setUpProject { + privilegeLevel: "readAndWrite" + }, (e, {@project_id, @user_id}) => + cb(e) + + (cb) => + FixturesManager.setUpDoc @project_id, {@lines, @version, @ops, @ranges}, (e, {@doc_id}) => + cb(e) + + (cb) => + @client = RealTimeClient.connect() + @client.on "connectionAccepted", cb + + (cb) => + @client.emit "joinProject", project_id: @project_id, cb + + (cb) => + @client.emit "joinDoc", @doc_id, @fromVersion, @options, (error, @returnedArgs...) => cb(error) + ], done + + it "should get the doc from the doc updater with the fromVersion", -> + MockDocUpdaterServer.getDocument + .calledWith(@project_id, @doc_id, @fromVersion) + .should.equal true + + it "should return the doc lines, version, ranges and ops", -> + @returnedArgs.should.deep.equal [@lines, @version, @ops, @ranges] + + it "should have joined the doc room", (done) -> + RealTimeClient.getConnectedClient @client.socket.sessionid, (error, client) => + expect(@doc_id in client.rooms).to.equal true + done() From d4c735c3eae8857f42ce7f18f2fb49c132865714 Mon Sep 17 00:00:00 2001 From: Shane Kilkelly Date: Tue, 26 Sep 2017 14:21:41 +0100 Subject: [PATCH 105/491] Pass anonymous-read token along as header to web-api --- services/real-time/app/coffee/Router.coffee | 3 +++ services/real-time/app/coffee/WebApiManager.coffee | 9 +++++++-- services/real-time/app/coffee/WebsocketController.coffee | 4 ++-- .../real-time/test/unit/coffee/WebApiManagerTests.coffee | 6 ++++-- .../test/unit/coffee/WebsocketControllerTests.coffee | 2 +- 5 files changed, 17 insertions(+), 7 deletions(-) diff --git a/services/real-time/app/coffee/Router.coffee b/services/real-time/app/coffee/Router.coffee index 2cc655eafc..1f7c744713 100644 --- a/services/real-time/app/coffee/Router.coffee +++ b/services/real-time/app/coffee/Router.coffee @@ -67,6 +67,9 @@ module.exports = Router = user = {_id: "anonymous-user"} client.on "joinProject", (data = {}, callback) -> + anonToken = session?.anonReadOnlyTokenAccess?[data.project_id] + if anonToken + user.anonToken = anonToken WebsocketController.joinProject client, user, data.project_id, (err, args...) -> if err? Router._handleError callback, err, client, "joinProject", {project_id: data.project_id, user_id: user?.id} diff --git a/services/real-time/app/coffee/WebApiManager.coffee b/services/real-time/app/coffee/WebApiManager.coffee index 63bb6604e0..0d24765b9f 100644 --- a/services/real-time/app/coffee/WebApiManager.coffee +++ b/services/real-time/app/coffee/WebApiManager.coffee @@ -3,9 +3,13 @@ settings = require "settings-sharelatex" logger = require "logger-sharelatex" module.exports = WebApiManager = - joinProject: (project_id, user_id, callback = (error, project, privilegeLevel) ->) -> + joinProject: (project_id, user, callback = (error, project, privilegeLevel) ->) -> + user_id = user._id logger.log {project_id, user_id}, "sending join project request to web" url = "#{settings.apis.web.url}/project/#{project_id}/join" + headers = {} + if user.anonToken? + headers['x-sl-anon-token'] = user.anonToken request.post { url: url qs: {user_id} @@ -15,6 +19,7 @@ module.exports = WebApiManager = sendImmediately: true json: true jar: false + headers: headers }, (error, response, data) -> return callback(error) if error? if 200 <= response.statusCode < 300 @@ -22,4 +27,4 @@ module.exports = WebApiManager = else err = new Error("non-success status code from web: #{response.statusCode}") logger.error {err, project_id, user_id}, "error accessing web api" - callback err \ No newline at end of file + callback err diff --git a/services/real-time/app/coffee/WebsocketController.coffee b/services/real-time/app/coffee/WebsocketController.coffee index e0242d4208..0da81b49a3 100644 --- a/services/real-time/app/coffee/WebsocketController.coffee +++ b/services/real-time/app/coffee/WebsocketController.coffee @@ -17,7 +17,7 @@ module.exports = WebsocketController = user_id = user?._id logger.log {user_id, project_id, client_id: client.id}, "user joining project" metrics.inc "editor.join-project" - WebApiManager.joinProject project_id, user_id, (error, project, privilegeLevel) -> + WebApiManager.joinProject project_id, user, (error, project, privilegeLevel) -> return callback(error) if error? if !privilegeLevel or privilegeLevel == "" @@ -205,4 +205,4 @@ module.exports = WebsocketController = for op in update.op if !op.c? return false - return true \ No newline at end of file + return true diff --git a/services/real-time/test/unit/coffee/WebApiManagerTests.coffee b/services/real-time/test/unit/coffee/WebApiManagerTests.coffee index 8ca08547df..453169cd54 100644 --- a/services/real-time/test/unit/coffee/WebApiManagerTests.coffee +++ b/services/real-time/test/unit/coffee/WebApiManagerTests.coffee @@ -8,6 +8,7 @@ describe 'WebApiManager', -> beforeEach -> @project_id = "project-id-123" @user_id = "user-id-123" + @user = {_id: @user_id} @callback = sinon.stub() @WebApiManager = SandboxedModule.require modulePath, requires: "request": @request = {} @@ -27,7 +28,7 @@ describe 'WebApiManager', -> privilegeLevel: "owner" } @request.post = sinon.stub().callsArgWith(1, null, {statusCode: 200}, @response) - @WebApiManager.joinProject @project_id, @user_id, @callback + @WebApiManager.joinProject @project_id, @user, @callback it "should send a request to web to join the project", -> @request.post @@ -41,6 +42,7 @@ describe 'WebApiManager', -> sendImmediately: true json: true jar: false + headers: {} }) .should.equal true @@ -58,4 +60,4 @@ describe 'WebApiManager', -> @callback .calledWith(new Error("non-success code from web: 500")) .should.equal true - \ No newline at end of file + diff --git a/services/real-time/test/unit/coffee/WebsocketControllerTests.coffee b/services/real-time/test/unit/coffee/WebsocketControllerTests.coffee index 8db81b716e..cddbc1a4c5 100644 --- a/services/real-time/test/unit/coffee/WebsocketControllerTests.coffee +++ b/services/real-time/test/unit/coffee/WebsocketControllerTests.coffee @@ -58,7 +58,7 @@ describe 'WebsocketController', -> it "should load the project from web", -> @WebApiManager.joinProject - .calledWith(@project_id, @user._id) + .calledWith(@project_id, @user) .should.equal true it "should join the project room", -> From 438bb28c04c29cf12acb256b0ad50b2f4c8ec2cc Mon Sep 17 00:00:00 2001 From: Shane Kilkelly Date: Fri, 29 Sep 2017 16:32:46 +0100 Subject: [PATCH 106/491] Get anonToken from joinProject payload --- services/real-time/app/coffee/Router.coffee | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/services/real-time/app/coffee/Router.coffee b/services/real-time/app/coffee/Router.coffee index 1f7c744713..74364f788f 100644 --- a/services/real-time/app/coffee/Router.coffee +++ b/services/real-time/app/coffee/Router.coffee @@ -67,9 +67,8 @@ module.exports = Router = user = {_id: "anonymous-user"} client.on "joinProject", (data = {}, callback) -> - anonToken = session?.anonReadOnlyTokenAccess?[data.project_id] - if anonToken - user.anonToken = anonToken + if data.anonToken + user.anonToken = data.anonToken WebsocketController.joinProject client, user, data.project_id, (err, args...) -> if err? Router._handleError callback, err, client, "joinProject", {project_id: data.project_id, user_id: user?.id} From ce2238c5d8242767ba8ab2537b428fb8100b10d3 Mon Sep 17 00:00:00 2001 From: Joe Green Date: Thu, 12 Oct 2017 16:56:13 +0100 Subject: [PATCH 107/491] only alert on master --- services/real-time/Jenkinsfile | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/services/real-time/Jenkinsfile b/services/real-time/Jenkinsfile index b684c49eb0..96ee723b26 100644 --- a/services/real-time/Jenkinsfile +++ b/services/real-time/Jenkinsfile @@ -70,6 +70,10 @@ pipeline { post { failure { + when { + branch 'master' + } + mail(from: "${EMAIL_ALERT_FROM}", to: "${EMAIL_ALERT_TO}", subject: "Jenkins build failed: ${JOB_NAME}:${BUILD_NUMBER}", From acc0193211dcdb35f67c36fcad793bdd0f8b8c69 Mon Sep 17 00:00:00 2001 From: Joe Green Date: Mon, 16 Oct 2017 14:13:20 +0100 Subject: [PATCH 108/491] Update Jenkinsfile --- services/real-time/Jenkinsfile | 4 ---- 1 file changed, 4 deletions(-) diff --git a/services/real-time/Jenkinsfile b/services/real-time/Jenkinsfile index 96ee723b26..b684c49eb0 100644 --- a/services/real-time/Jenkinsfile +++ b/services/real-time/Jenkinsfile @@ -70,10 +70,6 @@ pipeline { post { failure { - when { - branch 'master' - } - mail(from: "${EMAIL_ALERT_FROM}", to: "${EMAIL_ALERT_TO}", subject: "Jenkins build failed: ${JOB_NAME}:${BUILD_NUMBER}", From 76673d5f0aff81e885cedb34f0d2366a64d675b3 Mon Sep 17 00:00:00 2001 From: Shane Kilkelly Date: Fri, 20 Oct 2017 10:10:58 +0100 Subject: [PATCH 109/491] Change `anonToken` to `anonymousAccessToken` --- services/real-time/app/coffee/Router.coffee | 4 ++-- services/real-time/app/coffee/WebApiManager.coffee | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/services/real-time/app/coffee/Router.coffee b/services/real-time/app/coffee/Router.coffee index 74364f788f..57a0c4cc07 100644 --- a/services/real-time/app/coffee/Router.coffee +++ b/services/real-time/app/coffee/Router.coffee @@ -67,8 +67,8 @@ module.exports = Router = user = {_id: "anonymous-user"} client.on "joinProject", (data = {}, callback) -> - if data.anonToken - user.anonToken = data.anonToken + if data.anonymousAccessToken + user.anonymousAccessToken = data.anonymousAccessToken WebsocketController.joinProject client, user, data.project_id, (err, args...) -> if err? Router._handleError callback, err, client, "joinProject", {project_id: data.project_id, user_id: user?.id} diff --git a/services/real-time/app/coffee/WebApiManager.coffee b/services/real-time/app/coffee/WebApiManager.coffee index 0d24765b9f..35977c0a0f 100644 --- a/services/real-time/app/coffee/WebApiManager.coffee +++ b/services/real-time/app/coffee/WebApiManager.coffee @@ -8,8 +8,8 @@ module.exports = WebApiManager = logger.log {project_id, user_id}, "sending join project request to web" url = "#{settings.apis.web.url}/project/#{project_id}/join" headers = {} - if user.anonToken? - headers['x-sl-anon-token'] = user.anonToken + if user.anonymousAccessToken? + headers['x-sl-anonymous-access-token'] = user.anonToken request.post { url: url qs: {user_id} From 168d64632801a516ae9f253a0043e345d0596af2 Mon Sep 17 00:00:00 2001 From: Brian Gough Date: Fri, 20 Oct 2017 15:19:20 +0100 Subject: [PATCH 110/491] exit if mock servers fail to start --- .../acceptance/coffee/helpers/MockDocUpdaterServer.coffee | 6 +++++- .../test/acceptance/coffee/helpers/MockWebServer.coffee | 6 +++++- 2 files changed, 10 insertions(+), 2 deletions(-) diff --git a/services/real-time/test/acceptance/coffee/helpers/MockDocUpdaterServer.coffee b/services/real-time/test/acceptance/coffee/helpers/MockDocUpdaterServer.coffee index 0f196371b3..ac5bfc7093 100644 --- a/services/real-time/test/acceptance/coffee/helpers/MockDocUpdaterServer.coffee +++ b/services/real-time/test/acceptance/coffee/helpers/MockDocUpdaterServer.coffee @@ -38,5 +38,9 @@ module.exports = MockDocUpdaterServer = app.listen 3003, (error) -> MockDocUpdaterServer.running = true callback(error) + .on "error", (error) -> + console.error "error starting MockDocUpdaterServer:", error.message + process.exit(1) + -sinon.spy MockDocUpdaterServer, "getDocument" \ No newline at end of file +sinon.spy MockDocUpdaterServer, "getDocument" diff --git a/services/real-time/test/acceptance/coffee/helpers/MockWebServer.coffee b/services/real-time/test/acceptance/coffee/helpers/MockWebServer.coffee index 2fff23e252..06f52a6b19 100644 --- a/services/real-time/test/acceptance/coffee/helpers/MockWebServer.coffee +++ b/services/real-time/test/acceptance/coffee/helpers/MockWebServer.coffee @@ -35,5 +35,9 @@ module.exports = MockWebServer = app.listen 3000, (error) -> MockWebServer.running = true callback(error) + .on "error", (error) -> + console.error "error starting MockWebServer:", error.message + process.exit(1) + -sinon.spy MockWebServer, "joinProject" \ No newline at end of file +sinon.spy MockWebServer, "joinProject" From c5e602c0a9138ae71e27fe1638f9a1b0213352c9 Mon Sep 17 00:00:00 2001 From: Shane Kilkelly Date: Thu, 26 Oct 2017 16:00:06 +0100 Subject: [PATCH 111/491] Fix typo --- services/real-time/app/coffee/WebApiManager.coffee | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/services/real-time/app/coffee/WebApiManager.coffee b/services/real-time/app/coffee/WebApiManager.coffee index 35977c0a0f..f0e7526764 100644 --- a/services/real-time/app/coffee/WebApiManager.coffee +++ b/services/real-time/app/coffee/WebApiManager.coffee @@ -9,7 +9,7 @@ module.exports = WebApiManager = url = "#{settings.apis.web.url}/project/#{project_id}/join" headers = {} if user.anonymousAccessToken? - headers['x-sl-anonymous-access-token'] = user.anonToken + headers['x-sl-anonymous-access-token'] = user.anonymousAccessToken request.post { url: url qs: {user_id} From 510ee6bf4f894a33f34df855ff086e3995bf55c9 Mon Sep 17 00:00:00 2001 From: Brian Gough Date: Mon, 30 Oct 2017 13:43:36 +0000 Subject: [PATCH 112/491] use ioredis 3 via redis-sharelatex --- services/real-time/package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/services/real-time/package.json b/services/real-time/package.json index 4b3de3d5f3..15600c14e1 100644 --- a/services/real-time/package.json +++ b/services/real-time/package.json @@ -18,7 +18,7 @@ "ioredis": "^2.4.0", "logger-sharelatex": "git+https://github.com/sharelatex/logger-sharelatex.git#v1.1.0", "metrics-sharelatex": "git+https://github.com/sharelatex/metrics-sharelatex.git#v1.4.0", - "redis-sharelatex": "git+https://github.com/sharelatex/redis-sharelatex.git#v1.0.2", + "redis-sharelatex": "git+https://github.com/sharelatex/redis-sharelatex.git#v1.0.4", "request": "~2.34.0", "session.socket.io": "^0.1.6", "settings-sharelatex": "git+https://github.com/sharelatex/settings-sharelatex.git#v1.0.0", From cc118fa2304d33a159ece66575217d022c68e3ba Mon Sep 17 00:00:00 2001 From: Brian Gough Date: Mon, 30 Oct 2017 13:44:09 +0000 Subject: [PATCH 113/491] remove unnecessary ioredis package --- services/real-time/package.json | 1 - 1 file changed, 1 deletion(-) diff --git a/services/real-time/package.json b/services/real-time/package.json index 15600c14e1..cb097f8b78 100644 --- a/services/real-time/package.json +++ b/services/real-time/package.json @@ -15,7 +15,6 @@ "cookie-parser": "^1.3.3", "express": "^4.10.1", "express-session": "^1.9.1", - "ioredis": "^2.4.0", "logger-sharelatex": "git+https://github.com/sharelatex/logger-sharelatex.git#v1.1.0", "metrics-sharelatex": "git+https://github.com/sharelatex/metrics-sharelatex.git#v1.4.0", "redis-sharelatex": "git+https://github.com/sharelatex/redis-sharelatex.git#v1.0.4", From 0568aada17eef463ba7a70995b9f05880d8a1e0c Mon Sep 17 00:00:00 2001 From: Brian Gough Date: Mon, 30 Oct 2017 13:46:37 +0000 Subject: [PATCH 114/491] upgrade to node 6 --- services/real-time/.nvmrc | 2 +- services/real-time/Jenkinsfile | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/services/real-time/.nvmrc b/services/real-time/.nvmrc index d87edbfc10..26ec038c18 100644 --- a/services/real-time/.nvmrc +++ b/services/real-time/.nvmrc @@ -1 +1 @@ -4.2.1 \ No newline at end of file +6.9.5 \ No newline at end of file diff --git a/services/real-time/Jenkinsfile b/services/real-time/Jenkinsfile index b684c49eb0..d908d42063 100644 --- a/services/real-time/Jenkinsfile +++ b/services/real-time/Jenkinsfile @@ -15,7 +15,7 @@ pipeline { stage('Install') { agent { docker { - image 'node:4.2.1' + image 'node:6.9.5' args "-v /var/lib/jenkins/.npm:/tmp/.npm -e HOME=/tmp" reuseNode true } @@ -33,7 +33,7 @@ pipeline { stage('Compile and Test') { agent { docker { - image 'node:4.2.1' + image 'node:6.9.5' args "-v /var/lib/jenkins/.npm:/tmp/.npm -e HOME=/tmp" reuseNode true } From 009f8a3eae5e34bbea63d15ff2b09c02d568ffbe Mon Sep 17 00:00:00 2001 From: Brian Gough Date: Mon, 30 Oct 2017 16:14:29 +0000 Subject: [PATCH 115/491] add sentry support --- services/real-time/app.coffee | 2 ++ services/real-time/package.json | 2 +- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/services/real-time/app.coffee b/services/real-time/app.coffee index 0365f773c6..23ce867841 100644 --- a/services/real-time/app.coffee +++ b/services/real-time/app.coffee @@ -1,5 +1,7 @@ logger = require "logger-sharelatex" logger.initialize("real-time-sharelatex") +if Settings.sentry?.dsn? + logger.initializeErrorReporting(Settings.sentry.dsn) express = require("express") session = require("express-session") diff --git a/services/real-time/package.json b/services/real-time/package.json index cb097f8b78..8cfb35291a 100644 --- a/services/real-time/package.json +++ b/services/real-time/package.json @@ -15,7 +15,7 @@ "cookie-parser": "^1.3.3", "express": "^4.10.1", "express-session": "^1.9.1", - "logger-sharelatex": "git+https://github.com/sharelatex/logger-sharelatex.git#v1.1.0", + "logger-sharelatex": "git+https://github.com/sharelatex/logger-sharelatex.git#v1.5.6", "metrics-sharelatex": "git+https://github.com/sharelatex/metrics-sharelatex.git#v1.4.0", "redis-sharelatex": "git+https://github.com/sharelatex/redis-sharelatex.git#v1.0.4", "request": "~2.34.0", From 5c8291a8da7c9e1b8d3da60ed0b8c15f58ff7e9f Mon Sep 17 00:00:00 2001 From: Brian Gough Date: Tue, 6 Jun 2017 09:49:38 +0100 Subject: [PATCH 116/491] fix sentry initialisation --- services/real-time/app.coffee | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/services/real-time/app.coffee b/services/real-time/app.coffee index 23ce867841..824cdce5db 100644 --- a/services/real-time/app.coffee +++ b/services/real-time/app.coffee @@ -1,12 +1,12 @@ logger = require "logger-sharelatex" logger.initialize("real-time-sharelatex") -if Settings.sentry?.dsn? - logger.initializeErrorReporting(Settings.sentry.dsn) express = require("express") session = require("express-session") redis = require("redis-sharelatex") Settings = require "settings-sharelatex" +if Settings.sentry?.dsn? + logger.initializeErrorReporting(Settings.sentry.dsn) sessionRedisClient = redis.createClient(Settings.redis.websessions) From b734f7a3f7478a718323c8ada0327dec127954d8 Mon Sep 17 00:00:00 2001 From: Brian Gough Date: Fri, 10 Nov 2017 15:01:23 +0000 Subject: [PATCH 117/491] convert errors to warnings --- .../real-time/app/coffee/DocumentUpdaterController.coffee | 2 +- services/real-time/app/coffee/Router.coffee | 4 ++-- services/real-time/app/coffee/WebsocketController.coffee | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/services/real-time/app/coffee/DocumentUpdaterController.coffee b/services/real-time/app/coffee/DocumentUpdaterController.coffee index 01a8732a2a..451dc812bc 100644 --- a/services/real-time/app/coffee/DocumentUpdaterController.coffee +++ b/services/real-time/app/coffee/DocumentUpdaterController.coffee @@ -36,7 +36,7 @@ module.exports = DocumentUpdaterController = _processErrorFromDocumentUpdater: (io, doc_id, error, message) -> for client in io.sockets.clients(doc_id) - logger.error err: error, doc_id: doc_id, client_id: client.id, "error from document updater, disconnecting client" + logger.warn err: error, doc_id: doc_id, client_id: client.id, "error from document updater, disconnecting client" client.emit "otUpdateError", error, message client.disconnect() diff --git a/services/real-time/app/coffee/Router.coffee b/services/real-time/app/coffee/Router.coffee index d59853e831..ad74ebffa1 100644 --- a/services/real-time/app/coffee/Router.coffee +++ b/services/real-time/app/coffee/Router.coffee @@ -21,7 +21,7 @@ module.exports = Router = attrs[key] = value attrs.client_id = client.id attrs.err = error - if error.message in ["not authorized", "doc updater could not load requested ops"] + if error.message in ["not authorized", "doc updater could not load requested ops", "no project_id found on client"] logger.warn attrs, error.message return callback {message: error.message} else @@ -40,7 +40,7 @@ module.exports = Router = session.on 'connection', (error, client, session) -> if client? and error?.message?.match(/could not look up session by key/) - logger.err err: error, client: client?, session: session?, "invalid session" + logger.warn err: error, client: client?, session: session?, "invalid session" # tell the client to reauthenticate if it has an invalid session key client.emit("connectionRejected", {message: "invalid session"}) client.disconnect() diff --git a/services/real-time/app/coffee/WebsocketController.coffee b/services/real-time/app/coffee/WebsocketController.coffee index 77b797d560..abc50aedf0 100644 --- a/services/real-time/app/coffee/WebsocketController.coffee +++ b/services/real-time/app/coffee/WebsocketController.coffee @@ -182,7 +182,7 @@ module.exports = WebsocketController = return callback(new Error("no project_id found on client")) if !project_id? WebsocketController._assertClientCanApplyUpdate client, doc_id, update, (error) -> if error? - logger.error {err: error, doc_id, client_id: client.id, version: update.v}, "client is not authorized to make update" + logger.warn {err: error, doc_id, client_id: client.id, version: update.v}, "client is not authorized to make update" setTimeout () -> # Disconnect, but give the client the chance to receive the error client.disconnect() From c8ad33155158f22f1e09a6e54dabc8e49c251cd9 Mon Sep 17 00:00:00 2001 From: Brian Gough Date: Mon, 13 Nov 2017 09:41:04 +0000 Subject: [PATCH 118/491] fix unit tests --- .../test/unit/coffee/DocumentUpdaterControllerTests.coffee | 6 +++--- .../test/unit/coffee/WebsocketControllerTests.coffee | 6 +++--- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/services/real-time/test/unit/coffee/DocumentUpdaterControllerTests.coffee b/services/real-time/test/unit/coffee/DocumentUpdaterControllerTests.coffee index cb77a11513..fdd2f5fd5c 100644 --- a/services/real-time/test/unit/coffee/DocumentUpdaterControllerTests.coffee +++ b/services/real-time/test/unit/coffee/DocumentUpdaterControllerTests.coffee @@ -11,7 +11,7 @@ describe "DocumentUpdaterController", -> @callback = sinon.stub() @io = { "mock": "socket.io" } @EditorUpdatesController = SandboxedModule.require modulePath, requires: - "logger-sharelatex": @logger = { error: sinon.stub(), log: sinon.stub() } + "logger-sharelatex": @logger = { error: sinon.stub(), log: sinon.stub(), warn: sinon.stub() } "settings-sharelatex": @settings = redis: documentupdater: @@ -125,8 +125,8 @@ describe "DocumentUpdaterController", -> clients: sinon.stub().returns(@clients) @EditorUpdatesController._processErrorFromDocumentUpdater @io, @doc_id, "Something went wrong" - it "should log out an error", -> - @logger.error.called.should.equal true + it "should log a warning", -> + @logger.warn.called.should.equal true it "should disconnect all clients in that document", -> @io.sockets.clients.calledWith(@doc_id).should.equal true diff --git a/services/real-time/test/unit/coffee/WebsocketControllerTests.coffee b/services/real-time/test/unit/coffee/WebsocketControllerTests.coffee index dd220d254a..00f0638fb0 100644 --- a/services/real-time/test/unit/coffee/WebsocketControllerTests.coffee +++ b/services/real-time/test/unit/coffee/WebsocketControllerTests.coffee @@ -32,7 +32,7 @@ describe 'WebsocketController', -> "./DocumentUpdaterManager": @DocumentUpdaterManager = {} "./ConnectedUsersManager": @ConnectedUsersManager = {} "./WebsocketLoadBalancer": @WebsocketLoadBalancer = {} - "logger-sharelatex": @logger = { log: sinon.stub(), error: sinon.stub() } + "logger-sharelatex": @logger = { log: sinon.stub(), error: sinon.stub(), warn: sinon.stub() } "metrics-sharelatex": @metrics = inc: sinon.stub() set: sinon.stub() @@ -514,8 +514,8 @@ describe 'WebsocketController', -> # it "should disconnect the client", -> # @client.disconnect.called.should.equal true - it "should log an error", -> - @logger.error.called.should.equal true + it "should log a warning", -> + @logger.warn.called.should.equal true it "should call the callback with the error", -> @callback.calledWith(@error).should.equal true From 06c8729ce7451c5e583f18edf93fc6beb5d44e69 Mon Sep 17 00:00:00 2001 From: Shane Kilkelly Date: Tue, 12 Dec 2017 15:27:50 +0000 Subject: [PATCH 119/491] If a user has only their `first_name` set, don't label as Anonymous --- .../app/coffee/WebsocketController.coffee | 7 +++- .../coffee/WebsocketControllerTests.coffee | 40 +++++++++++++++++++ 2 files changed, 45 insertions(+), 2 deletions(-) diff --git a/services/real-time/app/coffee/WebsocketController.coffee b/services/real-time/app/coffee/WebsocketController.coffee index abc50aedf0..885524e3f2 100644 --- a/services/real-time/app/coffee/WebsocketController.coffee +++ b/services/real-time/app/coffee/WebsocketController.coffee @@ -145,8 +145,11 @@ module.exports = WebsocketController = cursorData.id = client.id cursorData.user_id = user_id if user_id? cursorData.email = email if email? - if first_name? and last_name? - cursorData.name = first_name + " " + last_name + if first_name? or last_name? + cursorData.name = if !last_name? + first_name + else + "#{first_name} #{last_name}" ConnectedUsersManager.updateUserPosition(project_id, client.id, { first_name: first_name, last_name: last_name, diff --git a/services/real-time/test/unit/coffee/WebsocketControllerTests.coffee b/services/real-time/test/unit/coffee/WebsocketControllerTests.coffee index 00f0638fb0..5427b9cd10 100644 --- a/services/real-time/test/unit/coffee/WebsocketControllerTests.coffee +++ b/services/real-time/test/unit/coffee/WebsocketControllerTests.coffee @@ -430,6 +430,46 @@ describe 'WebsocketController', -> it "should increment the update-client-position metric at 0.1 frequency", -> @metrics.inc.calledWith("editor.update-client-position", 0.1).should.equal true + describe "with a logged in user who has no last_name set", -> + beforeEach -> + @clientParams = { + project_id: @project_id + first_name: @first_name = "Douglas" + last_name: undefined + email: @email = "joe@example.com" + user_id: @user_id = "user-id-123" + } + @client.get = (param, callback) => callback null, @clientParams[param] + @WebsocketController.updateClientPosition @client, @update + + @populatedCursorData = + doc_id: @doc_id, + id: @client.id + name: "#{@first_name}" + row: @row + column: @column + email: @email + user_id: @user_id + + it "should send the update to the project room with the user's name", -> + @WebsocketLoadBalancer.emitToRoom.calledWith(@project_id, "clientTracking.clientUpdated", @populatedCursorData).should.equal true + + it "should send the cursor data to the connected user manager", (done)-> + @ConnectedUsersManager.updateUserPosition.calledWith(@project_id, @client.id, { + _id: @user_id, + email: @email, + first_name: @first_name, + last_name: undefined + }, { + row: @row + column: @column + doc_id: @doc_id + }).should.equal true + done() + + it "should increment the update-client-position metric at 0.1 frequency", -> + @metrics.inc.calledWith("editor.update-client-position", 0.1).should.equal true + describe "with an anonymous user", -> beforeEach -> @clientParams = { From 675814f1b1dbc4243da7e48e3c4a96e42cd9ff3d Mon Sep 17 00:00:00 2001 From: Shane Kilkelly Date: Wed, 13 Dec 2017 10:28:35 +0000 Subject: [PATCH 120/491] Handle the case where the user has only a last_name set --- .../app/coffee/WebsocketController.coffee | 8 ++-- .../coffee/WebsocketControllerTests.coffee | 41 +++++++++++++++++++ 2 files changed, 46 insertions(+), 3 deletions(-) diff --git a/services/real-time/app/coffee/WebsocketController.coffee b/services/real-time/app/coffee/WebsocketController.coffee index 885524e3f2..e0614eb153 100644 --- a/services/real-time/app/coffee/WebsocketController.coffee +++ b/services/real-time/app/coffee/WebsocketController.coffee @@ -146,10 +146,12 @@ module.exports = WebsocketController = cursorData.user_id = user_id if user_id? cursorData.email = email if email? if first_name? or last_name? - cursorData.name = if !last_name? - first_name - else + cursorData.name = if first_name && last_name "#{first_name} #{last_name}" + else if first_name + first_name + else if last_name + last_name ConnectedUsersManager.updateUserPosition(project_id, client.id, { first_name: first_name, last_name: last_name, diff --git a/services/real-time/test/unit/coffee/WebsocketControllerTests.coffee b/services/real-time/test/unit/coffee/WebsocketControllerTests.coffee index 5427b9cd10..b6a4069c67 100644 --- a/services/real-time/test/unit/coffee/WebsocketControllerTests.coffee +++ b/services/real-time/test/unit/coffee/WebsocketControllerTests.coffee @@ -470,6 +470,47 @@ describe 'WebsocketController', -> it "should increment the update-client-position metric at 0.1 frequency", -> @metrics.inc.calledWith("editor.update-client-position", 0.1).should.equal true + describe "with a logged in user who has no first_name set", -> + beforeEach -> + @clientParams = { + project_id: @project_id + first_name: undefined + last_name: @last_name = "Adams" + email: @email = "joe@example.com" + user_id: @user_id = "user-id-123" + } + @client.get = (param, callback) => callback null, @clientParams[param] + @WebsocketController.updateClientPosition @client, @update + + @populatedCursorData = + doc_id: @doc_id, + id: @client.id + name: "#{@last_name}" + row: @row + column: @column + email: @email + user_id: @user_id + + it "should send the update to the project room with the user's name", -> + @WebsocketLoadBalancer.emitToRoom.calledWith(@project_id, "clientTracking.clientUpdated", @populatedCursorData).should.equal true + + it "should send the cursor data to the connected user manager", (done)-> + @ConnectedUsersManager.updateUserPosition.calledWith(@project_id, @client.id, { + _id: @user_id, + email: @email, + first_name: undefined, + last_name: @last_name + }, { + row: @row + column: @column + doc_id: @doc_id + }).should.equal true + done() + + it "should increment the update-client-position metric at 0.1 frequency", -> + @metrics.inc.calledWith("editor.update-client-position", 0.1).should.equal true + + describe "with an anonymous user", -> beforeEach -> @clientParams = { From 7295342ec288830ef89361ace2fb195955305560 Mon Sep 17 00:00:00 2001 From: Shane Kilkelly Date: Mon, 18 Dec 2017 11:13:19 +0000 Subject: [PATCH 121/491] fix existance checks for first_name and last_name --- services/real-time/app/coffee/WebsocketController.coffee | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/services/real-time/app/coffee/WebsocketController.coffee b/services/real-time/app/coffee/WebsocketController.coffee index e0614eb153..b51e86c495 100644 --- a/services/real-time/app/coffee/WebsocketController.coffee +++ b/services/real-time/app/coffee/WebsocketController.coffee @@ -145,7 +145,7 @@ module.exports = WebsocketController = cursorData.id = client.id cursorData.user_id = user_id if user_id? cursorData.email = email if email? - if first_name? or last_name? + if first_name or last_name cursorData.name = if first_name && last_name "#{first_name} #{last_name}" else if first_name From 3d050f647bf463c27df5edb7786e43716cd5c890 Mon Sep 17 00:00:00 2001 From: James Allen Date: Fri, 29 Dec 2017 08:15:32 +0000 Subject: [PATCH 122/491] Provide hosts as environment settings and add npm run start script --- services/real-time/config/settings.defaults.coffee | 14 +++++++------- services/real-time/package.json | 4 ++++ 2 files changed, 11 insertions(+), 7 deletions(-) diff --git a/services/real-time/config/settings.defaults.coffee b/services/real-time/config/settings.defaults.coffee index d939dd55a8..e3464aa26a 100644 --- a/services/real-time/config/settings.defaults.coffee +++ b/services/real-time/config/settings.defaults.coffee @@ -1,7 +1,7 @@ module.exports = redis: realtime: - host: "localhost" + host: process.env['REDIS_HOST'] or "localhost" port: "6379" password: "" key_schema: @@ -9,35 +9,35 @@ module.exports = connectedUser: ({project_id, client_id})-> "connected_user:#{project_id}:#{client_id}" documentupdater: - host: "localhost" + host: process.env['REDIS_HOST'] or "localhost" port: "6379" password: "" key_schema: pendingUpdates: ({doc_id}) -> "PendingUpdates:#{doc_id}" websessions: - host: "localhost" + host: process.env['REDIS_HOST'] or "localhost" port: "6379" password: "" internal: realTime: port: 3026 - host: "localhost" + host: process.env['LISTEN_ADDRESS'] or "localhost" user: "sharelatex" pass: "password" apis: web: - url: "http://localhost:3000" + url: "http://#{process.env['WEB_HOST'] or "localhost"}:3000" user: "sharelatex" pass: "password" documentupdater: - url: "http://localhost:3003" + url: "http://#{process.env['DOCUPDATER_HOST'] or "localhost"}:3003" security: sessionSecret: "secret-please-change" - cookieName:"sharelatex.sid" + cookieName: "sharelatex.sid" max_doc_length: 2 * 1024 * 1024 # 2mb \ No newline at end of file diff --git a/services/real-time/package.json b/services/real-time/package.json index 8cfb35291a..fe057f7bda 100644 --- a/services/real-time/package.json +++ b/services/real-time/package.json @@ -7,6 +7,10 @@ "type": "git", "url": "https://github.com/sharelatex/real-time-sharelatex.git" }, + "scripts": { + "compile:app": "coffee -o app/js -c app/coffee && coffee -c app.coffee", + "start": "npm run compile:app && node app.js" + }, "dependencies": { "async": "^0.9.0", "basic-auth-connect": "^1.0.0", From 8c10c737dcf70d4e7ba727eba1926132bd6e7a39 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alberto=20Fern=C3=A1ndez=20Capel?= Date: Thu, 19 Apr 2018 14:37:46 +0100 Subject: [PATCH 123/491] Setup travis ci --- services/real-time/.travis.yml | 12 ++++++++++++ 1 file changed, 12 insertions(+) create mode 100644 services/real-time/.travis.yml diff --git a/services/real-time/.travis.yml b/services/real-time/.travis.yml new file mode 100644 index 0000000000..311f2fd9dd --- /dev/null +++ b/services/real-time/.travis.yml @@ -0,0 +1,12 @@ +language: node_js + +before_install: + - npm install -g grunt-cli + +install: + - npm install + - grunt install + +script: + - grunt test:unit + - grunt compile:acceptance_tests From 327fa79f1c7f2aefc5afa62c5a300cb418539c62 Mon Sep 17 00:00:00 2001 From: Henry Oswald Date: Wed, 23 May 2018 15:00:46 +0100 Subject: [PATCH 124/491] update build scripts, acceptence tests now pass. includes dockerfile 1.1.3 --- services/real-time/.dockerignore | 9 +++ services/real-time/.nvmrc | 2 +- services/real-time/Dockerfile | 22 ++++++ services/real-time/Jenkinsfile | 72 +++++++------------ services/real-time/Makefile | 42 +++++++++++ services/real-time/app.coffee | 2 + .../app/coffee/ConnectedUsersManager.coffee | 2 + .../coffee/DocumentUpdaterController.coffee | 2 + .../real-time/config/settings.defaults.coffee | 6 +- services/real-time/docker-compose.ci.yml | 32 +++++++++ services/real-time/docker-compose.yml | 41 +++++++++++ services/real-time/nodemon.json | 19 +++++ services/real-time/package.json | 13 +++- .../acceptance/coffee/ApplyUpdateTests.coffee | 2 +- .../coffee/ReceiveUpdateTests.coffee | 2 +- .../coffee/helpers/RealTimeClient.coffee | 2 +- .../coffee/helpers/RealtimeServer.coffee | 24 +++++++ 17 files changed, 239 insertions(+), 55 deletions(-) create mode 100644 services/real-time/.dockerignore create mode 100644 services/real-time/Dockerfile create mode 100644 services/real-time/Makefile create mode 100644 services/real-time/docker-compose.ci.yml create mode 100644 services/real-time/docker-compose.yml create mode 100644 services/real-time/nodemon.json create mode 100644 services/real-time/test/acceptance/coffee/helpers/RealtimeServer.coffee diff --git a/services/real-time/.dockerignore b/services/real-time/.dockerignore new file mode 100644 index 0000000000..386f26df30 --- /dev/null +++ b/services/real-time/.dockerignore @@ -0,0 +1,9 @@ +node_modules/* +gitrev +.git +.gitignore +.npm +.nvmrc +nodemon.json +app.js +**/js/* diff --git a/services/real-time/.nvmrc b/services/real-time/.nvmrc index 26ec038c18..e1e5d1369a 100644 --- a/services/real-time/.nvmrc +++ b/services/real-time/.nvmrc @@ -1 +1 @@ -6.9.5 \ No newline at end of file +6.9.5 diff --git a/services/real-time/Dockerfile b/services/real-time/Dockerfile new file mode 100644 index 0000000000..aabf01ad91 --- /dev/null +++ b/services/real-time/Dockerfile @@ -0,0 +1,22 @@ +FROM node:6.9.5 as app + +WORKDIR /app + +#wildcard as some files may not be in all repos +COPY package*.json npm-shrink*.json /app/ + +RUN npm install --quiet + +COPY . /app + + +RUN npm run compile:all + +FROM node:6.9.5 + +COPY --from=app /app /app + +WORKDIR /app +USER node + +CMD ["node","app.js"] diff --git a/services/real-time/Jenkinsfile b/services/real-time/Jenkinsfile index d908d42063..bc9ba0142f 100644 --- a/services/real-time/Jenkinsfile +++ b/services/real-time/Jenkinsfile @@ -1,87 +1,67 @@ -pipeline { - - agent any +String cron_string = BRANCH_NAME == "master" ? "@daily" : "" - environment { - HOME = "/tmp" - } +pipeline { + agent any triggers { pollSCM('* * * * *') - cron('@daily') + cron(cron_string) } stages { - stage('Install') { - agent { - docker { - image 'node:6.9.5' - args "-v /var/lib/jenkins/.npm:/tmp/.npm -e HOME=/tmp" - reuseNode true - } - } + stage('Build') { steps { - // we need to disable logallrefupdates, else git clones during the npm install will require git to lookup the user id - // which does not exist in the container's /etc/passwd file, causing the clone to fail. - sh 'git config --global core.logallrefupdates false' - sh 'rm -fr node_modules' - sh 'npm install' - sh 'npm rebuild' - sh 'npm install --quiet grunt-cli' + sh 'make build' } } - stage('Compile and Test') { - agent { - docker { - image 'node:6.9.5' - args "-v /var/lib/jenkins/.npm:/tmp/.npm -e HOME=/tmp" - reuseNode true - } - } + + stage('Unit Tests') { steps { - sh 'node_modules/.bin/grunt install' - sh 'node_modules/.bin/grunt compile:acceptance_tests' - sh 'node_modules/.bin/grunt test:unit' + sh 'DOCKER_COMPOSE_FLAGS="-f docker-compose.ci.yml" make test_unit' } } + stage('Acceptance Tests') { steps { - sh 'docker pull sharelatex/acceptance-test-runner' - sh 'docker run --rm -v $(pwd):/app sharelatex/acceptance-test-runner' + sh 'DOCKER_COMPOSE_FLAGS="-f docker-compose.ci.yml" make test_acceptance' } } - stage('Package') { + + stage('Package and publish build') { steps { - sh 'echo ${BUILD_NUMBER} > build_number.txt' - sh 'touch build.tar.gz' // Avoid tar warning about files changing during read - sh 'tar -czf build.tar.gz --exclude=build.tar.gz --exclude-vcs .' + sh 'make publish' } } - stage('Publish') { + + stage('Publish build number') { steps { + sh 'echo ${BRANCH_NAME}-${BUILD_NUMBER} > build_number.txt' withAWS(credentials:'S3_CI_BUILDS_AWS_KEYS', region:"${S3_REGION_BUILD_ARTEFACTS}") { - s3Upload(file:'build.tar.gz', bucket:"${S3_BUCKET_BUILD_ARTEFACTS}", path:"${JOB_NAME}/${BUILD_NUMBER}.tar.gz") // The deployment process uses this file to figure out the latest build s3Upload(file:'build_number.txt', bucket:"${S3_BUCKET_BUILD_ARTEFACTS}", path:"${JOB_NAME}/latest") } } } } - + post { + always { + sh 'DOCKER_COMPOSE_FLAGS="-f docker-compose.ci.yml" make test_clean' + } + failure { - mail(from: "${EMAIL_ALERT_FROM}", - to: "${EMAIL_ALERT_TO}", + mail(from: "${EMAIL_ALERT_FROM}", + to: "${EMAIL_ALERT_TO}", subject: "Jenkins build failed: ${JOB_NAME}:${BUILD_NUMBER}", body: "Build: ${BUILD_URL}") } } - + // The options directive is for configuration that applies to the whole job. options { // we'd like to make sure remove old builds, so we don't fill up our storage! buildDiscarder(logRotator(numToKeepStr:'50')) - + // And we'd really like to be sure that this build doesn't hang forever, so let's time it out after: timeout(time: 30, unit: 'MINUTES') } diff --git a/services/real-time/Makefile b/services/real-time/Makefile new file mode 100644 index 0000000000..48511326ad --- /dev/null +++ b/services/real-time/Makefile @@ -0,0 +1,42 @@ +# This file was auto-generated, do not edit it directly. +# Instead run bin/update_build_scripts from +# https://github.com/sharelatex/sharelatex-dev-environment +# Version: 1.1.3 + +BUILD_NUMBER ?= local +BRANCH_NAME ?= $(shell git rev-parse --abbrev-ref HEAD) +PROJECT_NAME = real-time +DOCKER_COMPOSE_FLAGS ?= -f docker-compose.yml +DOCKER_COMPOSE := BUILD_NUMBER=$(BUILD_NUMBER) \ + BRANCH_NAME=$(BRANCH_NAME) \ + PROJECT_NAME=$(PROJECT_NAME) \ + MOCHA_GREP=${MOCHA_GREP} \ + docker-compose ${DOCKER_COMPOSE_FLAGS} + + +clean: + rm -f app.js + rm -rf app/js + rm -rf test/unit/js + rm -rf test/acceptance/js + +test: test_unit test_acceptance + +test_unit: + @[ ! -d test/unit ] && echo "real-time has no unit tests" || $(DOCKER_COMPOSE) run --rm test_unit + +test_acceptance: test_clean test_acceptance_pre_run # clear the database before each acceptance test run + @[ ! -d test/acceptance ] && echo "real-time has no acceptance tests" || $(DOCKER_COMPOSE) run --rm test_acceptance + +test_clean: + $(DOCKER_COMPOSE) down -v -t 0 + +test_acceptance_pre_run: + @[ ! -f test/acceptance/scripts/pre-run ] && echo "real-time has no pre acceptance tests task" || $(DOCKER_COMPOSE) run --rm test_acceptance test/acceptance/scripts/pre-run +build: + docker build --pull --tag gcr.io/csh-gcdm-test/$(PROJECT_NAME):$(BRANCH_NAME)-$(BUILD_NUMBER) . + +publish: + docker push gcr.io/csh-gcdm-test/$(PROJECT_NAME):$(BRANCH_NAME)-$(BUILD_NUMBER) + +.PHONY: clean test test_unit test_acceptance test_clean build publish diff --git a/services/real-time/app.coffee b/services/real-time/app.coffee index 824cdce5db..91e427f3aa 100644 --- a/services/real-time/app.coffee +++ b/services/real-time/app.coffee @@ -8,6 +8,8 @@ Settings = require "settings-sharelatex" if Settings.sentry?.dsn? logger.initializeErrorReporting(Settings.sentry.dsn) +console.log "dasdsadasdsadsadsadsad" +console.log Settings.redis.websessions sessionRedisClient = redis.createClient(Settings.redis.websessions) RedisStore = require('connect-redis')(session) diff --git a/services/real-time/app/coffee/ConnectedUsersManager.coffee b/services/real-time/app/coffee/ConnectedUsersManager.coffee index 831a237356..38370da464 100644 --- a/services/real-time/app/coffee/ConnectedUsersManager.coffee +++ b/services/real-time/app/coffee/ConnectedUsersManager.coffee @@ -5,6 +5,8 @@ redis = require("redis-sharelatex") rclient = redis.createClient(Settings.redis.realtime) Keys = Settings.redis.realtime.key_schema +console.log Settings.redis.realtime, "REALTIME" + ONE_HOUR_IN_S = 60 * 60 ONE_DAY_IN_S = ONE_HOUR_IN_S * 24 FOUR_DAYS_IN_S = ONE_DAY_IN_S * 4 diff --git a/services/real-time/app/coffee/DocumentUpdaterController.coffee b/services/real-time/app/coffee/DocumentUpdaterController.coffee index 451dc812bc..05a70fe3a7 100644 --- a/services/real-time/app/coffee/DocumentUpdaterController.coffee +++ b/services/real-time/app/coffee/DocumentUpdaterController.coffee @@ -4,6 +4,8 @@ redis = require("redis-sharelatex") rclient = redis.createClient(settings.redis.documentupdater) SafeJsonParse = require "./SafeJsonParse" +console.log "REDIS", settings.redis + MESSAGE_SIZE_LOG_LIMIT = 1024 * 1024 # 1Mb module.exports = DocumentUpdaterController = diff --git a/services/real-time/config/settings.defaults.coffee b/services/real-time/config/settings.defaults.coffee index e3464aa26a..73e751a3b9 100644 --- a/services/real-time/config/settings.defaults.coffee +++ b/services/real-time/config/settings.defaults.coffee @@ -1,7 +1,7 @@ module.exports = redis: realtime: - host: process.env['REDIS_HOST'] or "localhost" + host: process.env['REDIS_HOST'] or "localhostssss" port: "6379" password: "" key_schema: @@ -9,14 +9,14 @@ module.exports = connectedUser: ({project_id, client_id})-> "connected_user:#{project_id}:#{client_id}" documentupdater: - host: process.env['REDIS_HOST'] or "localhost" + host: process.env['REDIS_HOST'] or "localhostssss" port: "6379" password: "" key_schema: pendingUpdates: ({doc_id}) -> "PendingUpdates:#{doc_id}" websessions: - host: process.env['REDIS_HOST'] or "localhost" + host: process.env['REDIS_HOST'] or "localhostssss" port: "6379" password: "" diff --git a/services/real-time/docker-compose.ci.yml b/services/real-time/docker-compose.ci.yml new file mode 100644 index 0000000000..21c006641e --- /dev/null +++ b/services/real-time/docker-compose.ci.yml @@ -0,0 +1,32 @@ +# This file was auto-generated, do not edit it directly. +# Instead run bin/update_build_scripts from +# https://github.com/sharelatex/sharelatex-dev-environment +# Version: 1.1.3 + +version: "2" + +services: + test_unit: + image: gcr.io/csh-gcdm-test/$PROJECT_NAME:$BRANCH_NAME-$BUILD_NUMBER + user: node + command: npm run test:unit:_run + + test_acceptance: + build: . + image: gcr.io/csh-gcdm-test/$PROJECT_NAME:$BRANCH_NAME-$BUILD_NUMBER + environment: + REDIS_HOST: redis + MONGO_HOST: mongo + POSTGRES_HOST: postgres + depends_on: + - mongo + - redis + user: node + command: npm run test:acceptance:_run + + redis: + image: redis + + mongo: + image: mongo:3.4 + diff --git a/services/real-time/docker-compose.yml b/services/real-time/docker-compose.yml new file mode 100644 index 0000000000..385872d7dd --- /dev/null +++ b/services/real-time/docker-compose.yml @@ -0,0 +1,41 @@ +# This file was auto-generated, do not edit it directly. +# Instead run bin/update_build_scripts from +# https://github.com/sharelatex/sharelatex-dev-environment +# Version: 1.1.3 + +version: "2" + +services: + + test_unit: + build: . + volumes: + - .:/app + working_dir: /app + environment: + MOCHA_GREP: ${MOCHA_GREP} + command: npm run test:unit + user: node + + test_acceptance: + build: . + volumes: + - .:/app + working_dir: /app + environment: + REDIS_HOST: redis + MONGO_HOST: mongo + POSTGRES_HOST: postgres + MOCHA_GREP: ${MOCHA_GREP} + user: node + depends_on: + - mongo + - redis + command: npm run test:acceptance + + redis: + image: redis + + mongo: + image: mongo:3.4 + diff --git a/services/real-time/nodemon.json b/services/real-time/nodemon.json new file mode 100644 index 0000000000..98db38d71b --- /dev/null +++ b/services/real-time/nodemon.json @@ -0,0 +1,19 @@ +{ + "ignore": [ + ".git", + "node_modules/" + ], + "verbose": true, + "legacyWatch": true, + "execMap": { + "js": "npm run start" + }, + + "watch": [ + "app/coffee/", + "app.coffee", + "config/" + ], + "ext": "coffee" + +} diff --git a/services/real-time/package.json b/services/real-time/package.json index fe057f7bda..30d4c81ef5 100644 --- a/services/real-time/package.json +++ b/services/real-time/package.json @@ -8,8 +8,16 @@ "url": "https://github.com/sharelatex/real-time-sharelatex.git" }, "scripts": { - "compile:app": "coffee -o app/js -c app/coffee && coffee -c app.coffee", - "start": "npm run compile:app && node app.js" + "compile:app": "([ -e app/coffee ] && coffee $COFFEE_OPTIONS -o app/js -c app/coffee || echo 'No CoffeeScript folder to compile') && ( [ -e app.coffee ] && coffee $COFFEE_OPTIONS -c app.coffee || echo 'No CoffeeScript app to compile')", + "start": "npm run compile:app && node $NODE_APP_OPTIONS app.js", + "test:acceptance:_run": "mocha --recursive --reporter spec --timeout 15000 --exit $@ test/acceptance/js", + "test:acceptance": "npm run compile:app && npm run compile:acceptance_tests && npm run test:acceptance:_run -- --grep=$MOCHA_GREP", + "test:unit:_run": "mocha --recursive --reporter spec $@ test/unit/js", + "test:unit": "npm run compile:app && npm run compile:unit_tests && npm run test:unit:_run -- --grep=$MOCHA_GREP", + "compile:unit_tests": "[ ! -e test/unit/coffee ] && echo 'No unit tests to compile' || coffee -o test/unit/js -c test/unit/coffee", + "compile:acceptance_tests": "[ ! -e test/acceptance/coffee ] && echo 'No acceptance tests to compile' || coffee -o test/acceptance/js -c test/acceptance/coffee", + "compile:all": "npm run compile:app && npm run compile:unit_tests && npm run compile:acceptance_tests", + "nodemon": "nodemon --config nodemon.json" }, "dependencies": { "async": "^0.9.0", @@ -42,6 +50,7 @@ "grunt-shell": "~0.7.0", "sandboxed-module": "~0.3.0", "sinon": "~1.5.2", + "mocha": "^4.0.1", "uid-safe": "^1.0.1", "timekeeper": "0.0.4" } diff --git a/services/real-time/test/acceptance/coffee/ApplyUpdateTests.coffee b/services/real-time/test/acceptance/coffee/ApplyUpdateTests.coffee index 0acc41cb70..2509f21108 100644 --- a/services/real-time/test/acceptance/coffee/ApplyUpdateTests.coffee +++ b/services/real-time/test/acceptance/coffee/ApplyUpdateTests.coffee @@ -8,7 +8,7 @@ FixturesManager = require "./helpers/FixturesManager" settings = require "settings-sharelatex" redis = require "redis-sharelatex" -rclient = redis.createClient(settings.redis.web) +rclient = redis.createClient(settings.redis.websessions) describe "applyOtUpdate", -> before -> diff --git a/services/real-time/test/acceptance/coffee/ReceiveUpdateTests.coffee b/services/real-time/test/acceptance/coffee/ReceiveUpdateTests.coffee index ec41598481..ec2c26ca4e 100644 --- a/services/real-time/test/acceptance/coffee/ReceiveUpdateTests.coffee +++ b/services/real-time/test/acceptance/coffee/ReceiveUpdateTests.coffee @@ -10,7 +10,7 @@ async = require "async" settings = require "settings-sharelatex" redis = require "redis-sharelatex" -rclient = redis.createClient(settings.redis.web) +rclient = redis.createClient(settings.redis.websessions) describe "receiveUpdate", -> before (done) -> diff --git a/services/real-time/test/acceptance/coffee/helpers/RealTimeClient.coffee b/services/real-time/test/acceptance/coffee/helpers/RealTimeClient.coffee index 52b869f862..21da045e83 100644 --- a/services/real-time/test/acceptance/coffee/helpers/RealTimeClient.coffee +++ b/services/real-time/test/acceptance/coffee/helpers/RealTimeClient.coffee @@ -4,7 +4,7 @@ io = require("socket.io-client") request = require "request" Settings = require "settings-sharelatex" redis = require "redis-sharelatex" -rclient = redis.createClient(Settings.redis.web) +rclient = redis.createClient(Settings.redis.websessions) uid = require('uid-safe').sync signature = require("cookie-signature") diff --git a/services/real-time/test/acceptance/coffee/helpers/RealtimeServer.coffee b/services/real-time/test/acceptance/coffee/helpers/RealtimeServer.coffee new file mode 100644 index 0000000000..12efd1ef13 --- /dev/null +++ b/services/real-time/test/acceptance/coffee/helpers/RealtimeServer.coffee @@ -0,0 +1,24 @@ +app = require('../../../../app') +require("logger-sharelatex").logger.level("info") +logger = require("logger-sharelatex") +Settings = require("settings-sharelatex") + +module.exports = + running: false + initing: false + callbacks: [] + ensureRunning: (callback = (error) ->) -> + if @running + return callback() + else if @initing + @callbacks.push callback + else + @initing = true + @callbacks.push callback + app.listen Settings.internal?.realtime?.port, "localhost", (error) => + throw error if error? + @running = true + logger.log("clsi running in dev mode") + + for callback in @callbacks + callback() \ No newline at end of file From 3a89bf0957dea12a10141da87e26a81b8984292c Mon Sep 17 00:00:00 2001 From: Henry Oswald Date: Wed, 3 Oct 2018 14:20:20 +0100 Subject: [PATCH 125/491] pass redis password through as env var --- services/real-time/config/settings.defaults.coffee | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/services/real-time/config/settings.defaults.coffee b/services/real-time/config/settings.defaults.coffee index 73e751a3b9..f8e76261a6 100644 --- a/services/real-time/config/settings.defaults.coffee +++ b/services/real-time/config/settings.defaults.coffee @@ -3,7 +3,7 @@ module.exports = realtime: host: process.env['REDIS_HOST'] or "localhostssss" port: "6379" - password: "" + password: process.env["REDIS_PASSWORD"] or "" key_schema: clientsInProject: ({project_id}) -> "clients_in_project:#{project_id}" connectedUser: ({project_id, client_id})-> "connected_user:#{project_id}:#{client_id}" @@ -11,14 +11,14 @@ module.exports = documentupdater: host: process.env['REDIS_HOST'] or "localhostssss" port: "6379" - password: "" + password: process.env["REDIS_PASSWORD"] or "" key_schema: pendingUpdates: ({doc_id}) -> "PendingUpdates:#{doc_id}" websessions: host: process.env['REDIS_HOST'] or "localhostssss" port: "6379" - password: "" + password: process.env["REDIS_PASSWORD"] or "" internal: realTime: From e322e46900c483e97b3ff3b21a6097d286f76796 Mon Sep 17 00:00:00 2001 From: Henry Oswald Date: Wed, 3 Oct 2018 14:21:02 +0100 Subject: [PATCH 126/491] update build scripts --- services/real-time/.github/ISSUE_TEMPLATE.md | 38 ++++++++++++++ .../.github/PULL_REQUEST_TEMPLATE.md | 45 ++++++++++++++++ services/real-time/Jenkinsfile | 51 ++++++++++++++++++- services/real-time/Makefile | 11 ++-- services/real-time/buildscript.txt | 9 ++++ services/real-time/docker-compose.ci.yml | 9 ++-- services/real-time/docker-compose.yml | 4 +- services/real-time/package.json | 9 ++-- 8 files changed, 162 insertions(+), 14 deletions(-) create mode 100644 services/real-time/.github/ISSUE_TEMPLATE.md create mode 100644 services/real-time/.github/PULL_REQUEST_TEMPLATE.md create mode 100644 services/real-time/buildscript.txt diff --git a/services/real-time/.github/ISSUE_TEMPLATE.md b/services/real-time/.github/ISSUE_TEMPLATE.md new file mode 100644 index 0000000000..e0093aa90c --- /dev/null +++ b/services/real-time/.github/ISSUE_TEMPLATE.md @@ -0,0 +1,38 @@ + + +## Steps to Reproduce + + + +1. +2. +3. + +## Expected Behaviour + + +## Observed Behaviour + + + +## Context + + +## Technical Info + + +* URL: +* Browser Name and version: +* Operating System and version (desktop or mobile): +* Signed in as: +* Project and/or file: + +## Analysis + + +## Who Needs to Know? + + + +- +- diff --git a/services/real-time/.github/PULL_REQUEST_TEMPLATE.md b/services/real-time/.github/PULL_REQUEST_TEMPLATE.md new file mode 100644 index 0000000000..ed25ee83c1 --- /dev/null +++ b/services/real-time/.github/PULL_REQUEST_TEMPLATE.md @@ -0,0 +1,45 @@ + + +### Description + + + +#### Screenshots + + + +#### Related Issues / PRs + + + +### Review + + + +#### Potential Impact + + + +#### Manual Testing Performed + +- [ ] +- [ ] + +#### Accessibility + + + +### Deployment + + + +#### Deployment Checklist + +- [ ] Update documentation not included in the PR (if any) +- [ ] + +#### Metrics and Monitoring + + + +#### Who Needs to Know? diff --git a/services/real-time/Jenkinsfile b/services/real-time/Jenkinsfile index bc9ba0142f..c553a7ef79 100644 --- a/services/real-time/Jenkinsfile +++ b/services/real-time/Jenkinsfile @@ -3,12 +3,33 @@ String cron_string = BRANCH_NAME == "master" ? "@daily" : "" pipeline { agent any + environment { + GIT_PROJECT = "real-time-sharelatex" + JENKINS_WORKFLOW = "real-time-sharelatex" + TARGET_URL = "${env.JENKINS_URL}blue/organizations/jenkins/${JENKINS_WORKFLOW}/detail/$BRANCH_NAME/$BUILD_NUMBER/pipeline" + GIT_API_URL = "https://api.github.com/repos/sharelatex/${GIT_PROJECT}/statuses/$GIT_COMMIT" + } + triggers { pollSCM('* * * * *') cron(cron_string) } stages { + stage('Install') { + steps { + withCredentials([usernamePassword(credentialsId: 'GITHUB_INTEGRATION', usernameVariable: 'GH_AUTH_USERNAME', passwordVariable: 'GH_AUTH_PASSWORD')]) { + sh "curl $GIT_API_URL \ + --data '{ \ + \"state\" : \"pending\", \ + \"target_url\": \"$TARGET_URL\", \ + \"description\": \"Your build is underway\", \ + \"context\": \"ci/jenkins\" }' \ + -u $GH_AUTH_USERNAME:$GH_AUTH_PASSWORD" + } + } + } + stage('Build') { steps { sh 'make build' @@ -29,7 +50,13 @@ pipeline { stage('Package and publish build') { steps { - sh 'make publish' + + withCredentials([file(credentialsId: 'gcr.io_overleaf-ops', variable: 'DOCKER_REPO_KEY_PATH')]) { + sh 'docker login -u _json_key --password-stdin https://gcr.io/overleaf-ops < ${DOCKER_REPO_KEY_PATH}' + } + sh 'DOCKER_REPO=gcr.io/overleaf-ops make publish' + sh 'docker logout https://gcr.io/overleaf-ops' + } } @@ -47,6 +74,19 @@ pipeline { post { always { sh 'DOCKER_COMPOSE_FLAGS="-f docker-compose.ci.yml" make test_clean' + sh 'make clean' + } + + success { + withCredentials([usernamePassword(credentialsId: 'GITHUB_INTEGRATION', usernameVariable: 'GH_AUTH_USERNAME', passwordVariable: 'GH_AUTH_PASSWORD')]) { + sh "curl $GIT_API_URL \ + --data '{ \ + \"state\" : \"success\", \ + \"target_url\": \"$TARGET_URL\", \ + \"description\": \"Your build succeeded!\", \ + \"context\": \"ci/jenkins\" }' \ + -u $GH_AUTH_USERNAME:$GH_AUTH_PASSWORD" + } } failure { @@ -54,6 +94,15 @@ pipeline { to: "${EMAIL_ALERT_TO}", subject: "Jenkins build failed: ${JOB_NAME}:${BUILD_NUMBER}", body: "Build: ${BUILD_URL}") + withCredentials([usernamePassword(credentialsId: 'GITHUB_INTEGRATION', usernameVariable: 'GH_AUTH_USERNAME', passwordVariable: 'GH_AUTH_PASSWORD')]) { + sh "curl $GIT_API_URL \ + --data '{ \ + \"state\" : \"failure\", \ + \"target_url\": \"$TARGET_URL\", \ + \"description\": \"Your build failed\", \ + \"context\": \"ci/jenkins\" }' \ + -u $GH_AUTH_USERNAME:$GH_AUTH_PASSWORD" + } } } diff --git a/services/real-time/Makefile b/services/real-time/Makefile index 48511326ad..7f5e935f7c 100644 --- a/services/real-time/Makefile +++ b/services/real-time/Makefile @@ -1,7 +1,7 @@ # This file was auto-generated, do not edit it directly. # Instead run bin/update_build_scripts from # https://github.com/sharelatex/sharelatex-dev-environment -# Version: 1.1.3 +# Version: 1.1.9 BUILD_NUMBER ?= local BRANCH_NAME ?= $(shell git rev-parse --abbrev-ref HEAD) @@ -15,6 +15,8 @@ DOCKER_COMPOSE := BUILD_NUMBER=$(BUILD_NUMBER) \ clean: + docker rmi ci/$(PROJECT_NAME):$(BRANCH_NAME)-$(BUILD_NUMBER) + docker rmi gcr.io/overleaf-ops/$(PROJECT_NAME):$(BRANCH_NAME)-$(BUILD_NUMBER) rm -f app.js rm -rf app/js rm -rf test/unit/js @@ -34,9 +36,12 @@ test_clean: test_acceptance_pre_run: @[ ! -f test/acceptance/scripts/pre-run ] && echo "real-time has no pre acceptance tests task" || $(DOCKER_COMPOSE) run --rm test_acceptance test/acceptance/scripts/pre-run build: - docker build --pull --tag gcr.io/csh-gcdm-test/$(PROJECT_NAME):$(BRANCH_NAME)-$(BUILD_NUMBER) . + docker build --pull --tag ci/$(PROJECT_NAME):$(BRANCH_NAME)-$(BUILD_NUMBER) \ + --tag gcr.io/overleaf-ops/$(PROJECT_NAME):$(BRANCH_NAME)-$(BUILD_NUMBER) \ + . publish: - docker push gcr.io/csh-gcdm-test/$(PROJECT_NAME):$(BRANCH_NAME)-$(BUILD_NUMBER) + + docker push $(DOCKER_REPO)/$(PROJECT_NAME):$(BRANCH_NAME)-$(BUILD_NUMBER) .PHONY: clean test test_unit test_acceptance test_clean build publish diff --git a/services/real-time/buildscript.txt b/services/real-time/buildscript.txt new file mode 100644 index 0000000000..dbdae4f5ba --- /dev/null +++ b/services/real-time/buildscript.txt @@ -0,0 +1,9 @@ +--script-version=1.1.9 +real-time +--node-version=6.9.5 +--acceptance-creds=None +--language=coffeescript +--dependencies=mongo,redis +--docker-repos=gcr.io/overleaf-ops +--kube=false +--build-target=docker diff --git a/services/real-time/docker-compose.ci.yml b/services/real-time/docker-compose.ci.yml index 21c006641e..17c4ddd2bf 100644 --- a/services/real-time/docker-compose.ci.yml +++ b/services/real-time/docker-compose.ci.yml @@ -1,23 +1,25 @@ # This file was auto-generated, do not edit it directly. # Instead run bin/update_build_scripts from # https://github.com/sharelatex/sharelatex-dev-environment -# Version: 1.1.3 +# Version: 1.1.9 version: "2" services: test_unit: - image: gcr.io/csh-gcdm-test/$PROJECT_NAME:$BRANCH_NAME-$BUILD_NUMBER + image: ci/$PROJECT_NAME:$BRANCH_NAME-$BUILD_NUMBER user: node command: npm run test:unit:_run test_acceptance: build: . - image: gcr.io/csh-gcdm-test/$PROJECT_NAME:$BRANCH_NAME-$BUILD_NUMBER + image: ci/$PROJECT_NAME:$BRANCH_NAME-$BUILD_NUMBER environment: + ELASTIC_SEARCH_DSN: es:9200 REDIS_HOST: redis MONGO_HOST: mongo POSTGRES_HOST: postgres + MOCHA_GREP: ${MOCHA_GREP} depends_on: - mongo - redis @@ -29,4 +31,3 @@ services: mongo: image: mongo:3.4 - diff --git a/services/real-time/docker-compose.yml b/services/real-time/docker-compose.yml index 385872d7dd..dcbc14e683 100644 --- a/services/real-time/docker-compose.yml +++ b/services/real-time/docker-compose.yml @@ -1,12 +1,11 @@ # This file was auto-generated, do not edit it directly. # Instead run bin/update_build_scripts from # https://github.com/sharelatex/sharelatex-dev-environment -# Version: 1.1.3 +# Version: 1.1.9 version: "2" services: - test_unit: build: . volumes: @@ -23,6 +22,7 @@ services: - .:/app working_dir: /app environment: + ELASTIC_SEARCH_DSN: es:9200 REDIS_HOST: redis MONGO_HOST: mongo POSTGRES_HOST: postgres diff --git a/services/real-time/package.json b/services/real-time/package.json index 30d4c81ef5..6e4f378c32 100644 --- a/services/real-time/package.json +++ b/services/real-time/package.json @@ -10,14 +10,15 @@ "scripts": { "compile:app": "([ -e app/coffee ] && coffee $COFFEE_OPTIONS -o app/js -c app/coffee || echo 'No CoffeeScript folder to compile') && ( [ -e app.coffee ] && coffee $COFFEE_OPTIONS -c app.coffee || echo 'No CoffeeScript app to compile')", "start": "npm run compile:app && node $NODE_APP_OPTIONS app.js", - "test:acceptance:_run": "mocha --recursive --reporter spec --timeout 15000 --exit $@ test/acceptance/js", + "test:acceptance:_run": "mocha --recursive --reporter spec --timeout 30000 --exit $@ test/acceptance/js", "test:acceptance": "npm run compile:app && npm run compile:acceptance_tests && npm run test:acceptance:_run -- --grep=$MOCHA_GREP", - "test:unit:_run": "mocha --recursive --reporter spec $@ test/unit/js", + "test:unit:_run": "mocha --recursive --reporter spec --exit $@ test/unit/js", "test:unit": "npm run compile:app && npm run compile:unit_tests && npm run test:unit:_run -- --grep=$MOCHA_GREP", "compile:unit_tests": "[ ! -e test/unit/coffee ] && echo 'No unit tests to compile' || coffee -o test/unit/js -c test/unit/coffee", "compile:acceptance_tests": "[ ! -e test/acceptance/coffee ] && echo 'No acceptance tests to compile' || coffee -o test/acceptance/js -c test/acceptance/coffee", - "compile:all": "npm run compile:app && npm run compile:unit_tests && npm run compile:acceptance_tests", - "nodemon": "nodemon --config nodemon.json" + "compile:all": "npm run compile:app && npm run compile:unit_tests && npm run compile:acceptance_tests && npm run compile:smoke_tests", + "nodemon": "nodemon --config nodemon.json", + "compile:smoke_tests": "[ ! -e test/smoke/coffee ] && echo 'No smoke tests to compile' || coffee -o test/smoke/js -c test/smoke/coffee" }, "dependencies": { "async": "^0.9.0", From ff8afb6c24af3fcbe4835de78eaf651b71325da7 Mon Sep 17 00:00:00 2001 From: Henry Oswald Date: Wed, 3 Oct 2018 15:01:31 +0100 Subject: [PATCH 127/491] pass redis port --- services/real-time/app.coffee | 2 -- services/real-time/config/settings.defaults.coffee | 6 +++--- 2 files changed, 3 insertions(+), 5 deletions(-) diff --git a/services/real-time/app.coffee b/services/real-time/app.coffee index 91e427f3aa..824cdce5db 100644 --- a/services/real-time/app.coffee +++ b/services/real-time/app.coffee @@ -8,8 +8,6 @@ Settings = require "settings-sharelatex" if Settings.sentry?.dsn? logger.initializeErrorReporting(Settings.sentry.dsn) -console.log "dasdsadasdsadsadsadsad" -console.log Settings.redis.websessions sessionRedisClient = redis.createClient(Settings.redis.websessions) RedisStore = require('connect-redis')(session) diff --git a/services/real-time/config/settings.defaults.coffee b/services/real-time/config/settings.defaults.coffee index f8e76261a6..152a6f058b 100644 --- a/services/real-time/config/settings.defaults.coffee +++ b/services/real-time/config/settings.defaults.coffee @@ -2,7 +2,7 @@ module.exports = redis: realtime: host: process.env['REDIS_HOST'] or "localhostssss" - port: "6379" + port: process.env['REDIS_PORT'] or "6379" password: process.env["REDIS_PASSWORD"] or "" key_schema: clientsInProject: ({project_id}) -> "clients_in_project:#{project_id}" @@ -10,14 +10,14 @@ module.exports = documentupdater: host: process.env['REDIS_HOST'] or "localhostssss" - port: "6379" + port: process.env['REDIS_PORT'] or "6379" password: process.env["REDIS_PASSWORD"] or "" key_schema: pendingUpdates: ({doc_id}) -> "PendingUpdates:#{doc_id}" websessions: host: process.env['REDIS_HOST'] or "localhostssss" - port: "6379" + port: process.env['REDIS_PORT'] or "6379" password: process.env["REDIS_PASSWORD"] or "" internal: From 4a495dbd1f438e597bff02846fa5e3a1785e8bd3 Mon Sep 17 00:00:00 2001 From: Alasdair Smith Date: Tue, 9 Oct 2018 11:53:49 +0100 Subject: [PATCH 128/491] Use setting instead of hard-coding port --- services/real-time/config/settings.defaults.coffee | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/services/real-time/config/settings.defaults.coffee b/services/real-time/config/settings.defaults.coffee index e3464aa26a..68a8a24e9e 100644 --- a/services/real-time/config/settings.defaults.coffee +++ b/services/real-time/config/settings.defaults.coffee @@ -29,7 +29,7 @@ module.exports = apis: web: - url: "http://#{process.env['WEB_HOST'] or "localhost"}:3000" + url: "http://#{process.env['WEB_HOST'] or "localhost"}:#{process.env['WEB_PORT'] or 3000}" user: "sharelatex" pass: "password" documentupdater: From 63d7bb501bbd34a7459ba7662fa5c75325e72569 Mon Sep 17 00:00:00 2001 From: Henry Oswald Date: Fri, 19 Oct 2018 16:44:40 +0100 Subject: [PATCH 129/491] return a 200 for root path for google health check --- services/real-time/app.coffee | 3 +++ 1 file changed, 3 insertions(+) diff --git a/services/real-time/app.coffee b/services/real-time/app.coffee index 824cdce5db..cde0c4af74 100644 --- a/services/real-time/app.coffee +++ b/services/real-time/app.coffee @@ -43,6 +43,9 @@ io.configure -> io.set('transports', ['websocket', 'flashsocket', 'htmlfile', 'xhr-polling', 'jsonp-polling']) io.set('log level', 1) +app.get "/", (req, res, next) -> + res.send "real-time-sharelatex is alive" + app.get "/status", (req, res, next) -> res.send "real-time-sharelatex is alive" From 477c446ea899e41b131057f092f1ac7973f231b2 Mon Sep 17 00:00:00 2001 From: Henry Oswald Date: Fri, 19 Oct 2018 19:28:05 +0100 Subject: [PATCH 130/491] add web-api host as ana option --- services/real-time/config/settings.defaults.coffee | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/services/real-time/config/settings.defaults.coffee b/services/real-time/config/settings.defaults.coffee index 152a6f058b..d052bc4e55 100644 --- a/services/real-time/config/settings.defaults.coffee +++ b/services/real-time/config/settings.defaults.coffee @@ -1,7 +1,7 @@ module.exports = redis: realtime: - host: process.env['REDIS_HOST'] or "localhostssss" + host: process.env['REDIS_HOST'] or "localhost" port: process.env['REDIS_PORT'] or "6379" password: process.env["REDIS_PASSWORD"] or "" key_schema: @@ -9,14 +9,14 @@ module.exports = connectedUser: ({project_id, client_id})-> "connected_user:#{project_id}:#{client_id}" documentupdater: - host: process.env['REDIS_HOST'] or "localhostssss" + host: process.env['REDIS_HOST'] or "localhost" port: process.env['REDIS_PORT'] or "6379" password: process.env["REDIS_PASSWORD"] or "" key_schema: pendingUpdates: ({doc_id}) -> "PendingUpdates:#{doc_id}" websessions: - host: process.env['REDIS_HOST'] or "localhostssss" + host: process.env['REDIS_HOST'] or "localhost" port: process.env['REDIS_PORT'] or "6379" password: process.env["REDIS_PASSWORD"] or "" @@ -29,7 +29,7 @@ module.exports = apis: web: - url: "http://#{process.env['WEB_HOST'] or "localhost"}:3000" + url: "http://#{process.env['WEB_API_HOST'] or process.env['WEB_HOST'] or "localhost"}:3000" user: "sharelatex" pass: "password" documentupdater: From bbc11ae1b54a10447f491af46c8fc73848cfc384 Mon Sep 17 00:00:00 2001 From: Henry Oswald Date: Fri, 19 Oct 2018 19:42:33 +0100 Subject: [PATCH 131/491] add DOCUMENT_UPDATER_HOST as option --- services/real-time/config/settings.defaults.coffee | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/services/real-time/config/settings.defaults.coffee b/services/real-time/config/settings.defaults.coffee index d052bc4e55..8d087c4a14 100644 --- a/services/real-time/config/settings.defaults.coffee +++ b/services/real-time/config/settings.defaults.coffee @@ -33,7 +33,7 @@ module.exports = user: "sharelatex" pass: "password" documentupdater: - url: "http://#{process.env['DOCUPDATER_HOST'] or "localhost"}:3003" + url: "http://#{process.env['DOCUMENT_UPDATER_HOST'] or process.env['DOCUPDATER_HOST'] or "localhost"}:3003" security: sessionSecret: "secret-please-change" From ca2af0af32a0ad35d45b9406f1b7089fb936471d Mon Sep 17 00:00:00 2001 From: Henry Oswald Date: Mon, 3 Dec 2018 14:05:03 +0000 Subject: [PATCH 132/491] make cookies configurable via env vars --- services/real-time/config/settings.defaults.coffee | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/services/real-time/config/settings.defaults.coffee b/services/real-time/config/settings.defaults.coffee index 8d087c4a14..485789cb12 100644 --- a/services/real-time/config/settings.defaults.coffee +++ b/services/real-time/config/settings.defaults.coffee @@ -36,8 +36,8 @@ module.exports = url: "http://#{process.env['DOCUMENT_UPDATER_HOST'] or process.env['DOCUPDATER_HOST'] or "localhost"}:3003" security: - sessionSecret: "secret-please-change" + sessionSecret: process.env['SESSION_SECRET'] or "secret-please-change" - cookieName: "sharelatex.sid" + cookieName: process.env['COOKIE_NAME'] or "sharelatex.sid" max_doc_length: 2 * 1024 * 1024 # 2mb \ No newline at end of file From 57cd7c734ac191179d74332b1726d74220266207 Mon Sep 17 00:00:00 2001 From: Henry Oswald Date: Mon, 3 Dec 2018 14:34:46 +0000 Subject: [PATCH 133/491] add debugging --- services/real-time/app.coffee | 2 ++ 1 file changed, 2 insertions(+) diff --git a/services/real-time/app.coffee b/services/real-time/app.coffee index cde0c4af74..b66375addb 100644 --- a/services/real-time/app.coffee +++ b/services/real-time/app.coffee @@ -27,6 +27,8 @@ io = require('socket.io').listen(server) # Bind to sessions sessionStore = new RedisStore(client: sessionRedisClient) cookieParser = CookieParser(Settings.security.sessionSecret) +console.log(Settings.security.sessionSecret, Settings.cookieName) +console.log(Settings) sessionSockets = new SessionSockets(io, sessionStore, cookieParser, Settings.cookieName) io.configure -> From 034e627a8a90203ae6fd498364a47d132d1030ba Mon Sep 17 00:00:00 2001 From: Henry Oswald Date: Tue, 4 Dec 2018 13:14:50 +0000 Subject: [PATCH 134/491] bump metrics and shrinkwrap --- services/real-time/Gruntfile.coffee | 75 - services/real-time/npm-shrinkwrap.json | 1951 ++++++++++++++++++++++++ services/real-time/package.json | 10 +- 3 files changed, 1952 insertions(+), 84 deletions(-) delete mode 100644 services/real-time/Gruntfile.coffee create mode 100644 services/real-time/npm-shrinkwrap.json diff --git a/services/real-time/Gruntfile.coffee b/services/real-time/Gruntfile.coffee deleted file mode 100644 index 28cc4233b0..0000000000 --- a/services/real-time/Gruntfile.coffee +++ /dev/null @@ -1,75 +0,0 @@ -module.exports = (grunt) -> - grunt.initConfig - forever: - app: - options: - index: "app.js" - coffee: - app_src: - expand: true, - flatten: true, - cwd: "app" - src: ['coffee/*.coffee'], - dest: 'app/js/', - ext: '.js' - - app: - src: "app.coffee" - dest: "app.js" - - unit_tests: - expand: true - cwd: "test/unit/coffee" - src: ["**/*.coffee"] - dest: "test/unit/js/" - ext: ".js" - - acceptance_tests: - expand: true - cwd: "test/acceptance/coffee" - src: ["**/*.coffee"] - dest: "test/acceptance/js/" - ext: ".js" - clean: - app: ["app/js/"] - unit_tests: ["test/unit/js"] - acceptance_tests: ["test/acceptance/js"] - smoke_tests: ["test/smoke/js"] - - execute: - app: - src: "app.js" - - mochaTest: - unit: - options: - reporter: grunt.option('reporter') or 'spec' - grep: grunt.option("grep") - src: ["test/unit/js/**/*.js"] - acceptance: - options: - reporter: grunt.option('reporter') or 'spec' - timeout: 40000 - grep: grunt.option("grep") - src: ["test/acceptance/js/**/*.js"] - - grunt.loadNpmTasks 'grunt-contrib-coffee' - grunt.loadNpmTasks 'grunt-contrib-clean' - grunt.loadNpmTasks 'grunt-mocha-test' - grunt.loadNpmTasks 'grunt-shell' - grunt.loadNpmTasks 'grunt-execute' - grunt.loadNpmTasks 'grunt-bunyan' - grunt.loadNpmTasks 'grunt-forever' - - grunt.registerTask 'compile:app', ['clean:app', 'coffee:app', 'coffee:app_src'] - grunt.registerTask 'run', ['compile:app', 'bunyan', 'execute'] - - grunt.registerTask 'compile:unit_tests', ['clean:unit_tests', 'coffee:unit_tests'] - grunt.registerTask 'test:unit', ['compile:app', 'compile:unit_tests', 'mochaTest:unit'] - - grunt.registerTask 'compile:acceptance_tests', ['clean:acceptance_tests', 'coffee:acceptance_tests'] - grunt.registerTask 'test:acceptance', ['compile:acceptance_tests', 'mochaTest:acceptance'] - - grunt.registerTask 'install', 'compile:app' - - grunt.registerTask 'default', ['run'] \ No newline at end of file diff --git a/services/real-time/npm-shrinkwrap.json b/services/real-time/npm-shrinkwrap.json new file mode 100644 index 0000000000..1da4679f14 --- /dev/null +++ b/services/real-time/npm-shrinkwrap.json @@ -0,0 +1,1951 @@ +{ + "name": "real-time-sharelatex", + "version": "0.1.4", + "dependencies": { + "@google-cloud/common": { + "version": "0.23.0", + "from": "@google-cloud/common@>=0.23.0 <0.24.0", + "resolved": "https://registry.npmjs.org/@google-cloud/common/-/common-0.23.0.tgz" + }, + "@google-cloud/debug-agent": { + "version": "3.0.0", + "from": "@google-cloud/debug-agent@>=3.0.0 <4.0.0", + "resolved": "https://registry.npmjs.org/@google-cloud/debug-agent/-/debug-agent-3.0.0.tgz", + "dependencies": { + "coffeescript": { + "version": "2.3.2", + "from": "coffeescript@>=2.0.0 <3.0.0", + "resolved": "https://registry.npmjs.org/coffeescript/-/coffeescript-2.3.2.tgz" + }, + "lodash": { + "version": "4.17.11", + "from": "lodash@>=4.12.0 <5.0.0", + "resolved": "https://registry.npmjs.org/lodash/-/lodash-4.17.11.tgz" + } + } + }, + "@google-cloud/projectify": { + "version": "0.3.2", + "from": "@google-cloud/projectify@>=0.3.0 <0.4.0", + "resolved": "https://registry.npmjs.org/@google-cloud/projectify/-/projectify-0.3.2.tgz" + }, + "@google-cloud/promisify": { + "version": "0.3.1", + "from": "@google-cloud/promisify@>=0.3.0 <0.4.0", + "resolved": "https://registry.npmjs.org/@google-cloud/promisify/-/promisify-0.3.1.tgz" + }, + "@google-cloud/trace-agent": { + "version": "3.4.0", + "from": "@google-cloud/trace-agent@>=3.2.0 <4.0.0", + "resolved": "https://registry.npmjs.org/@google-cloud/trace-agent/-/trace-agent-3.4.0.tgz", + "dependencies": { + "@google-cloud/common": { + "version": "0.27.0", + "from": "@google-cloud/common@>=0.27.0 <0.28.0", + "resolved": "https://registry.npmjs.org/@google-cloud/common/-/common-0.27.0.tgz" + }, + "gcp-metadata": { + "version": "0.9.0", + "from": "gcp-metadata@>=0.9.0 <0.10.0", + "resolved": "https://registry.npmjs.org/gcp-metadata/-/gcp-metadata-0.9.0.tgz" + }, + "google-auth-library": { + "version": "2.0.1", + "from": "google-auth-library@>=2.0.0 <3.0.0", + "resolved": "https://registry.npmjs.org/google-auth-library/-/google-auth-library-2.0.1.tgz", + "dependencies": { + "gcp-metadata": { + "version": "0.7.0", + "from": "gcp-metadata@^0.7.0", + "resolved": "https://registry.npmjs.org/gcp-metadata/-/gcp-metadata-0.7.0.tgz" + } + } + }, + "lru-cache": { + "version": "4.1.5", + "from": "lru-cache@^4.1.3", + "resolved": "https://registry.npmjs.org/lru-cache/-/lru-cache-4.1.5.tgz" + }, + "uuid": { + "version": "3.3.2", + "from": "uuid@^3.0.1", + "resolved": "https://registry.npmjs.org/uuid/-/uuid-3.3.2.tgz" + } + } + }, + "@sinonjs/formatio": { + "version": "2.0.0", + "from": "@sinonjs/formatio@>=2.0.0 <3.0.0", + "resolved": "https://registry.npmjs.org/@sinonjs/formatio/-/formatio-2.0.0.tgz" + }, + "@types/caseless": { + "version": "0.12.1", + "from": "@types/caseless@*", + "resolved": "https://registry.npmjs.org/@types/caseless/-/caseless-0.12.1.tgz" + }, + "@types/duplexify": { + "version": "3.6.0", + "from": "@types/duplexify@>=3.5.0 <4.0.0", + "resolved": "https://registry.npmjs.org/@types/duplexify/-/duplexify-3.6.0.tgz" + }, + "@types/form-data": { + "version": "2.2.1", + "from": "@types/form-data@*", + "resolved": "https://registry.npmjs.org/@types/form-data/-/form-data-2.2.1.tgz" + }, + "@types/node": { + "version": "10.12.12", + "from": "@types/node@*", + "resolved": "https://registry.npmjs.org/@types/node/-/node-10.12.12.tgz" + }, + "@types/request": { + "version": "2.48.1", + "from": "@types/request@>=2.47.0 <3.0.0", + "resolved": "https://registry.npmjs.org/@types/request/-/request-2.48.1.tgz" + }, + "@types/tough-cookie": { + "version": "2.3.4", + "from": "@types/tough-cookie@*", + "resolved": "https://registry.npmjs.org/@types/tough-cookie/-/tough-cookie-2.3.4.tgz" + }, + "abbrev": { + "version": "1.1.1", + "from": "abbrev@>=1.0.0 <2.0.0", + "resolved": "https://registry.npmjs.org/abbrev/-/abbrev-1.1.1.tgz" + }, + "accepts": { + "version": "1.3.5", + "from": "accepts@>=1.3.5 <1.4.0", + "resolved": "https://registry.npmjs.org/accepts/-/accepts-1.3.5.tgz" + }, + "acorn": { + "version": "5.7.3", + "from": "acorn@>=5.0.3 <6.0.0", + "resolved": "https://registry.npmjs.org/acorn/-/acorn-5.7.3.tgz" + }, + "active-x-obfuscator": { + "version": "0.0.1", + "from": "active-x-obfuscator@0.0.1", + "resolved": "https://registry.npmjs.org/active-x-obfuscator/-/active-x-obfuscator-0.0.1.tgz" + }, + "agent-base": { + "version": "4.2.1", + "from": "agent-base@>=4.1.0 <5.0.0", + "resolved": "https://registry.npmjs.org/agent-base/-/agent-base-4.2.1.tgz" + }, + "ansi-regex": { + "version": "0.2.1", + "from": "ansi-regex@>=0.2.0 <0.3.0", + "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-0.2.1.tgz" + }, + "ansi-styles": { + "version": "1.1.0", + "from": "ansi-styles@>=1.1.0 <2.0.0", + "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-1.1.0.tgz" + }, + "argparse": { + "version": "0.1.16", + "from": "argparse@>=0.1.11 <0.2.0", + "resolved": "https://registry.npmjs.org/argparse/-/argparse-0.1.16.tgz", + "dependencies": { + "underscore.string": { + "version": "2.4.0", + "from": "underscore.string@>=2.4.0 <2.5.0", + "resolved": "https://registry.npmjs.org/underscore.string/-/underscore.string-2.4.0.tgz" + } + } + }, + "array-flatten": { + "version": "1.1.1", + "from": "array-flatten@1.1.1", + "resolved": "https://registry.npmjs.org/array-flatten/-/array-flatten-1.1.1.tgz" + }, + "arrify": { + "version": "1.0.1", + "from": "arrify@>=1.0.1 <2.0.0", + "resolved": "https://registry.npmjs.org/arrify/-/arrify-1.0.1.tgz" + }, + "asn1": { + "version": "0.1.11", + "from": "asn1@0.1.11", + "resolved": "https://registry.npmjs.org/asn1/-/asn1-0.1.11.tgz", + "optional": true + }, + "assert-plus": { + "version": "0.1.5", + "from": "assert-plus@>=0.1.5 <0.2.0", + "resolved": "https://registry.npmjs.org/assert-plus/-/assert-plus-0.1.5.tgz", + "optional": true + }, + "async": { + "version": "0.9.2", + "from": "async@>=0.9.0 <0.10.0", + "resolved": "https://registry.npmjs.org/async/-/async-0.9.2.tgz" + }, + "async-listener": { + "version": "0.6.10", + "from": "async-listener@>=0.6.0 <0.7.0", + "resolved": "https://registry.npmjs.org/async-listener/-/async-listener-0.6.10.tgz" + }, + "aws-sign2": { + "version": "0.5.0", + "from": "aws-sign2@>=0.5.0 <0.6.0", + "resolved": "https://registry.npmjs.org/aws-sign2/-/aws-sign2-0.5.0.tgz", + "optional": true + }, + "axios": { + "version": "0.18.0", + "from": "axios@>=0.18.0 <0.19.0", + "resolved": "http://registry.npmjs.org/axios/-/axios-0.18.0.tgz" + }, + "balanced-match": { + "version": "1.0.0", + "from": "balanced-match@^1.0.0", + "resolved": "https://registry.npmjs.org/balanced-match/-/balanced-match-1.0.0.tgz" + }, + "base64id": { + "version": "0.1.0", + "from": "base64id@0.1.0", + "resolved": "https://registry.npmjs.org/base64id/-/base64id-0.1.0.tgz" + }, + "basic-auth-connect": { + "version": "1.0.0", + "from": "basic-auth-connect@>=1.0.0 <2.0.0", + "resolved": "https://registry.npmjs.org/basic-auth-connect/-/basic-auth-connect-1.0.0.tgz" + }, + "bignumber.js": { + "version": "7.2.1", + "from": "bignumber.js@>=7.0.0 <8.0.0", + "resolved": "https://registry.npmjs.org/bignumber.js/-/bignumber.js-7.2.1.tgz" + }, + "bintrees": { + "version": "1.0.1", + "from": "bintrees@1.0.1", + "resolved": "https://registry.npmjs.org/bintrees/-/bintrees-1.0.1.tgz" + }, + "bluebird": { + "version": "3.5.1", + "from": "bluebird@>=3.3.4 <4.0.0", + "resolved": "https://registry.npmjs.org/bluebird/-/bluebird-3.5.1.tgz" + }, + "body-parser": { + "version": "1.18.3", + "from": "body-parser@>=1.12.0 <2.0.0", + "resolved": "https://registry.npmjs.org/body-parser/-/body-parser-1.18.3.tgz" + }, + "boom": { + "version": "0.4.2", + "from": "boom@>=0.4.0 <0.5.0", + "resolved": "https://registry.npmjs.org/boom/-/boom-0.4.2.tgz" + }, + "brace-expansion": { + "version": "1.1.11", + "from": "brace-expansion@^1.1.7", + "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-1.1.11.tgz" + }, + "buffer-equal-constant-time": { + "version": "1.0.1", + "from": "buffer-equal-constant-time@1.0.1", + "resolved": "https://registry.npmjs.org/buffer-equal-constant-time/-/buffer-equal-constant-time-1.0.1.tgz" + }, + "builtin-modules": { + "version": "3.0.0", + "from": "builtin-modules@>=3.0.0 <4.0.0", + "resolved": "https://registry.npmjs.org/builtin-modules/-/builtin-modules-3.0.0.tgz" + }, + "bytes": { + "version": "3.0.0", + "from": "bytes@3.0.0", + "resolved": "https://registry.npmjs.org/bytes/-/bytes-3.0.0.tgz" + }, + "chalk": { + "version": "0.5.1", + "from": "chalk@>=0.5.0 <0.6.0", + "resolved": "https://registry.npmjs.org/chalk/-/chalk-0.5.1.tgz" + }, + "check-error": { + "version": "1.0.2", + "from": "check-error@>=1.0.1 <2.0.0", + "resolved": "https://registry.npmjs.org/check-error/-/check-error-1.0.2.tgz" + }, + "cluster-key-slot": { + "version": "1.0.9", + "from": "cluster-key-slot@>=1.0.6 <2.0.0", + "resolved": "https://registry.npmjs.org/cluster-key-slot/-/cluster-key-slot-1.0.9.tgz" + }, + "coffee-script": { + "version": "1.12.4", + "from": "coffee-script@1.12.4", + "resolved": "https://registry.npmjs.org/coffee-script/-/coffee-script-1.12.4.tgz" + }, + "colors": { + "version": "0.6.2", + "from": "colors@>=0.6.2 <0.7.0", + "resolved": "https://registry.npmjs.org/colors/-/colors-0.6.2.tgz" + }, + "combined-stream": { + "version": "0.0.7", + "from": "combined-stream@>=0.0.4 <0.1.0", + "resolved": "https://registry.npmjs.org/combined-stream/-/combined-stream-0.0.7.tgz", + "optional": true + }, + "commander": { + "version": "2.0.0", + "from": "commander@2.0.0", + "resolved": "http://registry.npmjs.org/commander/-/commander-2.0.0.tgz" + }, + "concat-map": { + "version": "0.0.1", + "from": "concat-map@0.0.1", + "resolved": "https://registry.npmjs.org/concat-map/-/concat-map-0.0.1.tgz" + }, + "connect-redis": { + "version": "2.5.1", + "from": "connect-redis@>=2.1.0 <3.0.0", + "resolved": "https://registry.npmjs.org/connect-redis/-/connect-redis-2.5.1.tgz", + "dependencies": { + "debug": { + "version": "1.0.5", + "from": "debug@>=1.0.4 <2.0.0", + "resolved": "https://registry.npmjs.org/debug/-/debug-1.0.5.tgz" + } + } + }, + "console-log-level": { + "version": "1.4.0", + "from": "console-log-level@>=1.4.0 <2.0.0", + "resolved": "https://registry.npmjs.org/console-log-level/-/console-log-level-1.4.0.tgz" + }, + "content-disposition": { + "version": "0.5.2", + "from": "content-disposition@0.5.2", + "resolved": "https://registry.npmjs.org/content-disposition/-/content-disposition-0.5.2.tgz" + }, + "content-type": { + "version": "1.0.4", + "from": "content-type@>=1.0.4 <1.1.0", + "resolved": "https://registry.npmjs.org/content-type/-/content-type-1.0.4.tgz" + }, + "continuation-local-storage": { + "version": "3.2.1", + "from": "continuation-local-storage@>=3.2.1 <4.0.0", + "resolved": "https://registry.npmjs.org/continuation-local-storage/-/continuation-local-storage-3.2.1.tgz" + }, + "cookie": { + "version": "0.3.1", + "from": "cookie@0.3.1", + "resolved": "https://registry.npmjs.org/cookie/-/cookie-0.3.1.tgz" + }, + "cookie-parser": { + "version": "1.4.3", + "from": "cookie-parser@>=1.3.3 <2.0.0", + "resolved": "https://registry.npmjs.org/cookie-parser/-/cookie-parser-1.4.3.tgz" + }, + "cookie-signature": { + "version": "1.0.6", + "from": "cookie-signature@1.0.6", + "resolved": "https://registry.npmjs.org/cookie-signature/-/cookie-signature-1.0.6.tgz" + }, + "core-util-is": { + "version": "1.0.2", + "from": "core-util-is@>=1.0.0 <1.1.0", + "resolved": "https://registry.npmjs.org/core-util-is/-/core-util-is-1.0.2.tgz" + }, + "crc": { + "version": "3.4.4", + "from": "crc@3.4.4", + "resolved": "https://registry.npmjs.org/crc/-/crc-3.4.4.tgz" + }, + "cryptiles": { + "version": "0.2.2", + "from": "cryptiles@>=0.2.0 <0.3.0", + "resolved": "https://registry.npmjs.org/cryptiles/-/cryptiles-0.2.2.tgz", + "optional": true + }, + "ctype": { + "version": "0.5.3", + "from": "ctype@0.5.3", + "resolved": "https://registry.npmjs.org/ctype/-/ctype-0.5.3.tgz", + "optional": true + }, + "dateformat": { + "version": "1.0.2-1.2.3", + "from": "dateformat@1.0.2-1.2.3", + "resolved": "https://registry.npmjs.org/dateformat/-/dateformat-1.0.2-1.2.3.tgz" + }, + "debug": { + "version": "2.6.9", + "from": "debug@2.6.9", + "resolved": "https://registry.npmjs.org/debug/-/debug-2.6.9.tgz" + }, + "define-properties": { + "version": "1.1.3", + "from": "define-properties@>=1.1.2 <2.0.0", + "resolved": "https://registry.npmjs.org/define-properties/-/define-properties-1.1.3.tgz" + }, + "delayed-stream": { + "version": "0.0.5", + "from": "delayed-stream@0.0.5", + "resolved": "https://registry.npmjs.org/delayed-stream/-/delayed-stream-0.0.5.tgz", + "optional": true + }, + "denque": { + "version": "1.2.3", + "from": "denque@>=1.1.0 <2.0.0", + "resolved": "https://registry.npmjs.org/denque/-/denque-1.2.3.tgz" + }, + "depd": { + "version": "1.1.2", + "from": "depd@>=1.1.2 <1.2.0", + "resolved": "https://registry.npmjs.org/depd/-/depd-1.1.2.tgz" + }, + "destroy": { + "version": "1.0.4", + "from": "destroy@>=1.0.4 <1.1.0", + "resolved": "https://registry.npmjs.org/destroy/-/destroy-1.0.4.tgz" + }, + "diff": { + "version": "1.0.7", + "from": "diff@1.0.7", + "resolved": "https://registry.npmjs.org/diff/-/diff-1.0.7.tgz" + }, + "duplexify": { + "version": "3.6.1", + "from": "duplexify@>=3.6.0 <4.0.0", + "resolved": "https://registry.npmjs.org/duplexify/-/duplexify-3.6.1.tgz" + }, + "ecdsa-sig-formatter": { + "version": "1.0.10", + "from": "ecdsa-sig-formatter@1.0.10", + "resolved": "https://registry.npmjs.org/ecdsa-sig-formatter/-/ecdsa-sig-formatter-1.0.10.tgz" + }, + "ee-first": { + "version": "1.1.1", + "from": "ee-first@1.1.1", + "resolved": "https://registry.npmjs.org/ee-first/-/ee-first-1.1.1.tgz" + }, + "emitter-listener": { + "version": "1.1.2", + "from": "emitter-listener@>=1.1.1 <2.0.0", + "resolved": "https://registry.npmjs.org/emitter-listener/-/emitter-listener-1.1.2.tgz" + }, + "encodeurl": { + "version": "1.0.2", + "from": "encodeurl@>=1.0.2 <1.1.0", + "resolved": "https://registry.npmjs.org/encodeurl/-/encodeurl-1.0.2.tgz" + }, + "end-of-stream": { + "version": "1.4.1", + "from": "end-of-stream@>=1.0.0 <2.0.0", + "resolved": "https://registry.npmjs.org/end-of-stream/-/end-of-stream-1.4.1.tgz" + }, + "ent": { + "version": "2.2.0", + "from": "ent@>=2.2.0 <3.0.0", + "resolved": "https://registry.npmjs.org/ent/-/ent-2.2.0.tgz" + }, + "es-abstract": { + "version": "1.12.0", + "from": "es-abstract@>=1.5.1 <2.0.0", + "resolved": "https://registry.npmjs.org/es-abstract/-/es-abstract-1.12.0.tgz" + }, + "es-to-primitive": { + "version": "1.2.0", + "from": "es-to-primitive@>=1.1.1 <2.0.0", + "resolved": "https://registry.npmjs.org/es-to-primitive/-/es-to-primitive-1.2.0.tgz" + }, + "es6-promise": { + "version": "4.2.5", + "from": "es6-promise@>=4.0.3 <5.0.0", + "resolved": "https://registry.npmjs.org/es6-promise/-/es6-promise-4.2.5.tgz" + }, + "es6-promisify": { + "version": "5.0.0", + "from": "es6-promisify@>=5.0.0 <6.0.0", + "resolved": "https://registry.npmjs.org/es6-promisify/-/es6-promisify-5.0.0.tgz" + }, + "escape-html": { + "version": "1.0.3", + "from": "escape-html@>=1.0.3 <1.1.0", + "resolved": "https://registry.npmjs.org/escape-html/-/escape-html-1.0.3.tgz" + }, + "escape-string-regexp": { + "version": "1.0.5", + "from": "escape-string-regexp@>=1.0.0 <2.0.0", + "resolved": "https://registry.npmjs.org/escape-string-regexp/-/escape-string-regexp-1.0.5.tgz" + }, + "esprima": { + "version": "1.0.4", + "from": "esprima@>=1.0.2 <1.1.0", + "resolved": "https://registry.npmjs.org/esprima/-/esprima-1.0.4.tgz" + }, + "etag": { + "version": "1.8.1", + "from": "etag@>=1.8.1 <1.9.0", + "resolved": "https://registry.npmjs.org/etag/-/etag-1.8.1.tgz" + }, + "eventemitter2": { + "version": "0.4.14", + "from": "eventemitter2@>=0.4.13 <0.5.0", + "resolved": "https://registry.npmjs.org/eventemitter2/-/eventemitter2-0.4.14.tgz" + }, + "exit": { + "version": "0.1.2", + "from": "exit@>=0.1.1 <0.2.0", + "resolved": "https://registry.npmjs.org/exit/-/exit-0.1.2.tgz" + }, + "express": { + "version": "4.16.3", + "from": "express@>=4.10.1 <5.0.0", + "resolved": "https://registry.npmjs.org/express/-/express-4.16.3.tgz", + "dependencies": { + "body-parser": { + "version": "1.18.2", + "from": "body-parser@1.18.2", + "resolved": "https://registry.npmjs.org/body-parser/-/body-parser-1.18.2.tgz" + }, + "iconv-lite": { + "version": "0.4.19", + "from": "iconv-lite@0.4.19", + "resolved": "https://registry.npmjs.org/iconv-lite/-/iconv-lite-0.4.19.tgz" + }, + "qs": { + "version": "6.5.1", + "from": "qs@6.5.1", + "resolved": "https://registry.npmjs.org/qs/-/qs-6.5.1.tgz" + }, + "raw-body": { + "version": "2.3.2", + "from": "raw-body@2.3.2", + "resolved": "https://registry.npmjs.org/raw-body/-/raw-body-2.3.2.tgz", + "dependencies": { + "depd": { + "version": "1.1.1", + "from": "depd@1.1.1", + "resolved": "https://registry.npmjs.org/depd/-/depd-1.1.1.tgz" + }, + "http-errors": { + "version": "1.6.2", + "from": "http-errors@1.6.2", + "resolved": "https://registry.npmjs.org/http-errors/-/http-errors-1.6.2.tgz" + }, + "setprototypeof": { + "version": "1.0.3", + "from": "setprototypeof@1.0.3", + "resolved": "https://registry.npmjs.org/setprototypeof/-/setprototypeof-1.0.3.tgz" + } + } + }, + "statuses": { + "version": "1.4.0", + "from": "statuses@>=1.4.0 <1.5.0", + "resolved": "https://registry.npmjs.org/statuses/-/statuses-1.4.0.tgz" + } + } + }, + "express-session": { + "version": "1.15.6", + "from": "express-session@>=1.9.1 <2.0.0", + "resolved": "https://registry.npmjs.org/express-session/-/express-session-1.15.6.tgz", + "dependencies": { + "uid-safe": { + "version": "2.1.5", + "from": "uid-safe@>=2.1.5 <2.2.0", + "resolved": "https://registry.npmjs.org/uid-safe/-/uid-safe-2.1.5.tgz" + } + } + }, + "extend": { + "version": "3.0.2", + "from": "extend@>=3.0.1 <4.0.0", + "resolved": "https://registry.npmjs.org/extend/-/extend-3.0.2.tgz" + }, + "finalhandler": { + "version": "1.1.1", + "from": "finalhandler@1.1.1", + "resolved": "https://registry.npmjs.org/finalhandler/-/finalhandler-1.1.1.tgz", + "dependencies": { + "statuses": { + "version": "1.4.0", + "from": "statuses@>=1.4.0 <1.5.0", + "resolved": "https://registry.npmjs.org/statuses/-/statuses-1.4.0.tgz" + } + } + }, + "findit2": { + "version": "2.2.3", + "from": "findit2@>=2.2.3 <3.0.0", + "resolved": "https://registry.npmjs.org/findit2/-/findit2-2.2.3.tgz" + }, + "findup-sync": { + "version": "0.1.3", + "from": "findup-sync@>=0.1.2 <0.2.0", + "resolved": "https://registry.npmjs.org/findup-sync/-/findup-sync-0.1.3.tgz", + "dependencies": { + "glob": { + "version": "3.2.11", + "from": "glob@>=3.2.9 <3.3.0", + "resolved": "https://registry.npmjs.org/glob/-/glob-3.2.11.tgz" + }, + "lodash": { + "version": "2.4.2", + "from": "lodash@>=2.4.1 <2.5.0", + "resolved": "https://registry.npmjs.org/lodash/-/lodash-2.4.2.tgz" + }, + "minimatch": { + "version": "0.3.0", + "from": "minimatch@>=0.3.0 <0.4.0", + "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-0.3.0.tgz" + } + } + }, + "flexbuffer": { + "version": "0.0.6", + "from": "flexbuffer@0.0.6", + "resolved": "https://registry.npmjs.org/flexbuffer/-/flexbuffer-0.0.6.tgz" + }, + "follow-redirects": { + "version": "1.5.10", + "from": "follow-redirects@>=1.3.0 <2.0.0", + "resolved": "https://registry.npmjs.org/follow-redirects/-/follow-redirects-1.5.10.tgz", + "dependencies": { + "debug": { + "version": "3.1.0", + "from": "debug@3.1.0", + "resolved": "https://registry.npmjs.org/debug/-/debug-3.1.0.tgz" + } + } + }, + "forever-agent": { + "version": "0.5.2", + "from": "forever-agent@>=0.5.0 <0.6.0", + "resolved": "https://registry.npmjs.org/forever-agent/-/forever-agent-0.5.2.tgz" + }, + "form-data": { + "version": "0.1.4", + "from": "form-data@>=0.1.0 <0.2.0", + "resolved": "https://registry.npmjs.org/form-data/-/form-data-0.1.4.tgz", + "optional": true, + "dependencies": { + "mime": { + "version": "1.2.11", + "from": "mime@>=1.2.11 <1.3.0", + "resolved": "https://registry.npmjs.org/mime/-/mime-1.2.11.tgz", + "optional": true + } + } + }, + "forwarded": { + "version": "0.1.2", + "from": "forwarded@>=0.1.2 <0.2.0", + "resolved": "https://registry.npmjs.org/forwarded/-/forwarded-0.1.2.tgz" + }, + "fresh": { + "version": "0.5.2", + "from": "fresh@0.5.2", + "resolved": "https://registry.npmjs.org/fresh/-/fresh-0.5.2.tgz" + }, + "fs-extra": { + "version": "0.9.1", + "from": "fs-extra@>=0.9.1 <0.10.0", + "resolved": "https://registry.npmjs.org/fs-extra/-/fs-extra-0.9.1.tgz", + "dependencies": { + "ncp": { + "version": "0.5.1", + "from": "ncp@>=0.5.1 <0.6.0", + "resolved": "https://registry.npmjs.org/ncp/-/ncp-0.5.1.tgz" + } + } + }, + "function-bind": { + "version": "1.1.1", + "from": "function-bind@>=1.1.1 <2.0.0", + "resolved": "https://registry.npmjs.org/function-bind/-/function-bind-1.1.1.tgz" + }, + "gcp-metadata": { + "version": "0.7.0", + "from": "gcp-metadata@>=0.7.0 <0.8.0", + "resolved": "https://registry.npmjs.org/gcp-metadata/-/gcp-metadata-0.7.0.tgz" + }, + "get-func-name": { + "version": "2.0.0", + "from": "get-func-name@>=2.0.0 <3.0.0", + "resolved": "https://registry.npmjs.org/get-func-name/-/get-func-name-2.0.0.tgz" + }, + "getobject": { + "version": "0.1.0", + "from": "getobject@>=0.1.0 <0.2.0", + "resolved": "https://registry.npmjs.org/getobject/-/getobject-0.1.0.tgz" + }, + "glob": { + "version": "6.0.4", + "from": "glob@^6.0.1", + "resolved": "https://registry.npmjs.org/glob/-/glob-6.0.4.tgz" + }, + "google-auth-library": { + "version": "1.6.1", + "from": "google-auth-library@>=1.6.0 <2.0.0", + "resolved": "https://registry.npmjs.org/google-auth-library/-/google-auth-library-1.6.1.tgz", + "dependencies": { + "gcp-metadata": { + "version": "0.6.3", + "from": "gcp-metadata@>=0.6.3 <0.7.0", + "resolved": "https://registry.npmjs.org/gcp-metadata/-/gcp-metadata-0.6.3.tgz" + }, + "lru-cache": { + "version": "4.1.5", + "from": "lru-cache@>=4.1.3 <5.0.0", + "resolved": "https://registry.npmjs.org/lru-cache/-/lru-cache-4.1.5.tgz" + } + } + }, + "google-p12-pem": { + "version": "1.0.2", + "from": "google-p12-pem@>=1.0.0 <2.0.0", + "resolved": "https://registry.npmjs.org/google-p12-pem/-/google-p12-pem-1.0.2.tgz", + "dependencies": { + "pify": { + "version": "3.0.0", + "from": "pify@>=3.0.0 <4.0.0", + "resolved": "https://registry.npmjs.org/pify/-/pify-3.0.0.tgz" + } + } + }, + "graceful-fs": { + "version": "1.2.3", + "from": "graceful-fs@>=1.2.0 <1.3.0", + "resolved": "https://registry.npmjs.org/graceful-fs/-/graceful-fs-1.2.3.tgz" + }, + "growl": { + "version": "1.7.0", + "from": "growl@>=1.7.0 <1.8.0", + "resolved": "https://registry.npmjs.org/growl/-/growl-1.7.0.tgz" + }, + "grunt": { + "version": "0.4.5", + "from": "grunt@>=0.4.5 <0.5.0", + "resolved": "https://registry.npmjs.org/grunt/-/grunt-0.4.5.tgz", + "dependencies": { + "async": { + "version": "0.1.22", + "from": "async@>=0.1.22 <0.2.0", + "resolved": "https://registry.npmjs.org/async/-/async-0.1.22.tgz" + }, + "coffee-script": { + "version": "1.3.3", + "from": "coffee-script@>=1.3.3 <1.4.0", + "resolved": "https://registry.npmjs.org/coffee-script/-/coffee-script-1.3.3.tgz" + }, + "glob": { + "version": "3.1.21", + "from": "glob@>=3.1.21 <3.2.0", + "resolved": "https://registry.npmjs.org/glob/-/glob-3.1.21.tgz" + }, + "iconv-lite": { + "version": "0.2.11", + "from": "iconv-lite@>=0.2.11 <0.3.0", + "resolved": "https://registry.npmjs.org/iconv-lite/-/iconv-lite-0.2.11.tgz" + }, + "inherits": { + "version": "1.0.2", + "from": "inherits@>=1.0.0 <2.0.0", + "resolved": "https://registry.npmjs.org/inherits/-/inherits-1.0.2.tgz" + }, + "minimatch": { + "version": "0.2.14", + "from": "minimatch@>=0.2.12 <0.3.0", + "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-0.2.14.tgz" + }, + "rimraf": { + "version": "2.2.8", + "from": "rimraf@>=2.2.8 <2.3.0", + "resolved": "https://registry.npmjs.org/rimraf/-/rimraf-2.2.8.tgz" + } + } + }, + "grunt-bunyan": { + "version": "0.5.0", + "from": "grunt-bunyan@>=0.5.0 <0.6.0", + "resolved": "https://registry.npmjs.org/grunt-bunyan/-/grunt-bunyan-0.5.0.tgz", + "dependencies": { + "lodash": { + "version": "2.4.2", + "from": "lodash@>=2.4.1 <2.5.0", + "resolved": "https://registry.npmjs.org/lodash/-/lodash-2.4.2.tgz" + } + } + }, + "grunt-contrib-clean": { + "version": "0.6.0", + "from": "grunt-contrib-clean@>=0.6.0 <0.7.0", + "resolved": "https://registry.npmjs.org/grunt-contrib-clean/-/grunt-contrib-clean-0.6.0.tgz", + "dependencies": { + "rimraf": { + "version": "2.2.8", + "from": "rimraf@>=2.2.1 <2.3.0", + "resolved": "https://registry.npmjs.org/rimraf/-/rimraf-2.2.8.tgz" + } + } + }, + "grunt-contrib-coffee": { + "version": "0.11.1", + "from": "grunt-contrib-coffee@>=0.11.0 <0.12.0", + "resolved": "https://registry.npmjs.org/grunt-contrib-coffee/-/grunt-contrib-coffee-0.11.1.tgz", + "dependencies": { + "coffee-script": { + "version": "1.7.1", + "from": "coffee-script@>=1.7.0 <1.8.0", + "resolved": "https://registry.npmjs.org/coffee-script/-/coffee-script-1.7.1.tgz" + }, + "lodash": { + "version": "2.4.2", + "from": "lodash@>=2.4.1 <2.5.0", + "resolved": "https://registry.npmjs.org/lodash/-/lodash-2.4.2.tgz" + }, + "mkdirp": { + "version": "0.3.5", + "from": "mkdirp@>=0.3.5 <0.4.0", + "resolved": "https://registry.npmjs.org/mkdirp/-/mkdirp-0.3.5.tgz" + } + } + }, + "grunt-execute": { + "version": "0.2.2", + "from": "grunt-execute@>=0.2.2 <0.3.0", + "resolved": "https://registry.npmjs.org/grunt-execute/-/grunt-execute-0.2.2.tgz" + }, + "grunt-legacy-log": { + "version": "0.1.3", + "from": "grunt-legacy-log@>=0.1.0 <0.2.0", + "resolved": "https://registry.npmjs.org/grunt-legacy-log/-/grunt-legacy-log-0.1.3.tgz", + "dependencies": { + "lodash": { + "version": "2.4.2", + "from": "lodash@>=2.4.1 <2.5.0", + "resolved": "https://registry.npmjs.org/lodash/-/lodash-2.4.2.tgz" + }, + "underscore.string": { + "version": "2.3.3", + "from": "underscore.string@>=2.3.3 <2.4.0", + "resolved": "https://registry.npmjs.org/underscore.string/-/underscore.string-2.3.3.tgz" + } + } + }, + "grunt-legacy-log-utils": { + "version": "0.1.1", + "from": "grunt-legacy-log-utils@>=0.1.1 <0.2.0", + "resolved": "https://registry.npmjs.org/grunt-legacy-log-utils/-/grunt-legacy-log-utils-0.1.1.tgz", + "dependencies": { + "lodash": { + "version": "2.4.2", + "from": "lodash@>=2.4.1 <2.5.0", + "resolved": "https://registry.npmjs.org/lodash/-/lodash-2.4.2.tgz" + }, + "underscore.string": { + "version": "2.3.3", + "from": "underscore.string@>=2.3.3 <2.4.0", + "resolved": "https://registry.npmjs.org/underscore.string/-/underscore.string-2.3.3.tgz" + } + } + }, + "grunt-legacy-util": { + "version": "0.2.0", + "from": "grunt-legacy-util@>=0.2.0 <0.3.0", + "resolved": "https://registry.npmjs.org/grunt-legacy-util/-/grunt-legacy-util-0.2.0.tgz", + "dependencies": { + "async": { + "version": "0.1.22", + "from": "async@>=0.1.22 <0.2.0", + "resolved": "https://registry.npmjs.org/async/-/async-0.1.22.tgz" + } + } + }, + "grunt-mocha-test": { + "version": "0.11.0", + "from": "grunt-mocha-test@>=0.11.0 <0.12.0", + "resolved": "https://registry.npmjs.org/grunt-mocha-test/-/grunt-mocha-test-0.11.0.tgz", + "dependencies": { + "glob": { + "version": "3.2.3", + "from": "glob@3.2.3", + "resolved": "https://registry.npmjs.org/glob/-/glob-3.2.3.tgz" + }, + "graceful-fs": { + "version": "2.0.3", + "from": "graceful-fs@>=2.0.0 <2.1.0", + "resolved": "https://registry.npmjs.org/graceful-fs/-/graceful-fs-2.0.3.tgz" + }, + "minimatch": { + "version": "0.2.14", + "from": "minimatch@>=0.2.11 <0.3.0", + "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-0.2.14.tgz" + }, + "mkdirp": { + "version": "0.3.5", + "from": "mkdirp@0.3.5", + "resolved": "http://registry.npmjs.org/mkdirp/-/mkdirp-0.3.5.tgz" + }, + "mocha": { + "version": "1.20.1", + "from": "mocha@>=1.20.0 <1.21.0", + "resolved": "https://registry.npmjs.org/mocha/-/mocha-1.20.1.tgz" + } + } + }, + "gtoken": { + "version": "2.3.0", + "from": "gtoken@>=2.3.0 <3.0.0", + "resolved": "https://registry.npmjs.org/gtoken/-/gtoken-2.3.0.tgz", + "dependencies": { + "mime": { + "version": "2.4.0", + "from": "mime@>=2.2.0 <3.0.0", + "resolved": "https://registry.npmjs.org/mime/-/mime-2.4.0.tgz" + }, + "pify": { + "version": "3.0.0", + "from": "pify@>=3.0.0 <4.0.0", + "resolved": "https://registry.npmjs.org/pify/-/pify-3.0.0.tgz" + } + } + }, + "has": { + "version": "1.0.3", + "from": "has@>=1.0.1 <2.0.0", + "resolved": "https://registry.npmjs.org/has/-/has-1.0.3.tgz" + }, + "has-ansi": { + "version": "0.1.0", + "from": "has-ansi@>=0.1.0 <0.2.0", + "resolved": "https://registry.npmjs.org/has-ansi/-/has-ansi-0.1.0.tgz" + }, + "has-flag": { + "version": "3.0.0", + "from": "has-flag@>=3.0.0 <4.0.0", + "resolved": "https://registry.npmjs.org/has-flag/-/has-flag-3.0.0.tgz" + }, + "has-symbols": { + "version": "1.0.0", + "from": "has-symbols@>=1.0.0 <2.0.0", + "resolved": "https://registry.npmjs.org/has-symbols/-/has-symbols-1.0.0.tgz" + }, + "hawk": { + "version": "1.0.0", + "from": "hawk@>=1.0.0 <1.1.0", + "resolved": "https://registry.npmjs.org/hawk/-/hawk-1.0.0.tgz", + "optional": true + }, + "hex2dec": { + "version": "1.1.1", + "from": "hex2dec@>=1.0.1 <2.0.0", + "resolved": "https://registry.npmjs.org/hex2dec/-/hex2dec-1.1.1.tgz" + }, + "hoek": { + "version": "0.9.1", + "from": "hoek@>=0.9.0 <0.10.0", + "resolved": "https://registry.npmjs.org/hoek/-/hoek-0.9.1.tgz" + }, + "hooker": { + "version": "0.2.3", + "from": "hooker@>=0.2.3 <0.3.0", + "resolved": "https://registry.npmjs.org/hooker/-/hooker-0.2.3.tgz" + }, + "http-errors": { + "version": "1.6.3", + "from": "http-errors@>=1.6.3 <1.7.0", + "resolved": "https://registry.npmjs.org/http-errors/-/http-errors-1.6.3.tgz" + }, + "http-signature": { + "version": "0.10.1", + "from": "http-signature@>=0.10.0 <0.11.0", + "resolved": "https://registry.npmjs.org/http-signature/-/http-signature-0.10.1.tgz", + "optional": true + }, + "https-proxy-agent": { + "version": "2.2.1", + "from": "https-proxy-agent@>=2.2.1 <3.0.0", + "resolved": "https://registry.npmjs.org/https-proxy-agent/-/https-proxy-agent-2.2.1.tgz", + "dependencies": { + "debug": { + "version": "3.2.6", + "from": "debug@>=3.1.0 <4.0.0", + "resolved": "https://registry.npmjs.org/debug/-/debug-3.2.6.tgz" + }, + "ms": { + "version": "2.1.1", + "from": "ms@>=2.1.1 <3.0.0", + "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.1.tgz" + } + } + }, + "iconv-lite": { + "version": "0.4.23", + "from": "iconv-lite@0.4.23", + "resolved": "https://registry.npmjs.org/iconv-lite/-/iconv-lite-0.4.23.tgz" + }, + "inflight": { + "version": "1.0.6", + "from": "inflight@^1.0.4", + "resolved": "https://registry.npmjs.org/inflight/-/inflight-1.0.6.tgz" + }, + "inherits": { + "version": "2.0.3", + "from": "inherits@2.0.3", + "resolved": "https://registry.npmjs.org/inherits/-/inherits-2.0.3.tgz" + }, + "ioredis": { + "version": "3.2.2", + "from": "ioredis@>=3.2.1 <4.0.0", + "resolved": "https://registry.npmjs.org/ioredis/-/ioredis-3.2.2.tgz" + }, + "ipaddr.js": { + "version": "1.6.0", + "from": "ipaddr.js@1.6.0", + "resolved": "https://registry.npmjs.org/ipaddr.js/-/ipaddr.js-1.6.0.tgz" + }, + "is": { + "version": "3.2.1", + "from": "is@>=3.2.1 <4.0.0", + "resolved": "https://registry.npmjs.org/is/-/is-3.2.1.tgz" + }, + "is-buffer": { + "version": "1.1.6", + "from": "is-buffer@>=1.1.5 <2.0.0", + "resolved": "https://registry.npmjs.org/is-buffer/-/is-buffer-1.1.6.tgz" + }, + "is-callable": { + "version": "1.1.4", + "from": "is-callable@>=1.1.3 <2.0.0", + "resolved": "https://registry.npmjs.org/is-callable/-/is-callable-1.1.4.tgz" + }, + "is-date-object": { + "version": "1.0.1", + "from": "is-date-object@>=1.0.1 <2.0.0", + "resolved": "https://registry.npmjs.org/is-date-object/-/is-date-object-1.0.1.tgz" + }, + "is-regex": { + "version": "1.0.4", + "from": "is-regex@>=1.0.4 <2.0.0", + "resolved": "https://registry.npmjs.org/is-regex/-/is-regex-1.0.4.tgz" + }, + "is-symbol": { + "version": "1.0.2", + "from": "is-symbol@>=1.0.2 <2.0.0", + "resolved": "https://registry.npmjs.org/is-symbol/-/is-symbol-1.0.2.tgz" + }, + "isarray": { + "version": "0.0.1", + "from": "isarray@0.0.1", + "resolved": "https://registry.npmjs.org/isarray/-/isarray-0.0.1.tgz" + }, + "jade": { + "version": "0.26.3", + "from": "jade@0.26.3", + "resolved": "https://registry.npmjs.org/jade/-/jade-0.26.3.tgz", + "dependencies": { + "commander": { + "version": "0.6.1", + "from": "commander@0.6.1", + "resolved": "https://registry.npmjs.org/commander/-/commander-0.6.1.tgz" + }, + "mkdirp": { + "version": "0.3.0", + "from": "mkdirp@0.3.0", + "resolved": "https://registry.npmjs.org/mkdirp/-/mkdirp-0.3.0.tgz" + } + } + }, + "js-yaml": { + "version": "2.0.5", + "from": "js-yaml@>=2.0.5 <2.1.0", + "resolved": "https://registry.npmjs.org/js-yaml/-/js-yaml-2.0.5.tgz" + }, + "json-bigint": { + "version": "0.3.0", + "from": "json-bigint@>=0.3.0 <0.4.0", + "resolved": "https://registry.npmjs.org/json-bigint/-/json-bigint-0.3.0.tgz" + }, + "json-stringify-safe": { + "version": "5.0.1", + "from": "json-stringify-safe@5.0.1", + "resolved": "https://registry.npmjs.org/json-stringify-safe/-/json-stringify-safe-5.0.1.tgz" + }, + "jsonfile": { + "version": "1.1.1", + "from": "jsonfile@~1.1.0", + "resolved": "https://registry.npmjs.org/jsonfile/-/jsonfile-1.1.1.tgz" + }, + "just-extend": { + "version": "1.1.27", + "from": "just-extend@>=1.1.27 <2.0.0", + "resolved": "https://registry.npmjs.org/just-extend/-/just-extend-1.1.27.tgz" + }, + "jwa": { + "version": "1.1.6", + "from": "jwa@>=1.1.5 <2.0.0", + "resolved": "https://registry.npmjs.org/jwa/-/jwa-1.1.6.tgz" + }, + "jws": { + "version": "3.1.5", + "from": "jws@>=3.1.5 <4.0.0", + "resolved": "https://registry.npmjs.org/jws/-/jws-3.1.5.tgz" + }, + "lodash": { + "version": "0.9.2", + "from": "lodash@>=0.9.2 <0.10.0", + "resolved": "https://registry.npmjs.org/lodash/-/lodash-0.9.2.tgz" + }, + "lodash.assign": { + "version": "4.2.0", + "from": "lodash.assign@>=4.2.0 <5.0.0", + "resolved": "https://registry.npmjs.org/lodash.assign/-/lodash.assign-4.2.0.tgz" + }, + "lodash.bind": { + "version": "4.2.1", + "from": "lodash.bind@>=4.2.1 <5.0.0", + "resolved": "https://registry.npmjs.org/lodash.bind/-/lodash.bind-4.2.1.tgz" + }, + "lodash.clone": { + "version": "4.5.0", + "from": "lodash.clone@>=4.5.0 <5.0.0", + "resolved": "https://registry.npmjs.org/lodash.clone/-/lodash.clone-4.5.0.tgz" + }, + "lodash.clonedeep": { + "version": "4.5.0", + "from": "lodash.clonedeep@>=4.5.0 <5.0.0", + "resolved": "https://registry.npmjs.org/lodash.clonedeep/-/lodash.clonedeep-4.5.0.tgz" + }, + "lodash.defaults": { + "version": "4.2.0", + "from": "lodash.defaults@>=4.2.0 <5.0.0", + "resolved": "https://registry.npmjs.org/lodash.defaults/-/lodash.defaults-4.2.0.tgz" + }, + "lodash.difference": { + "version": "4.5.0", + "from": "lodash.difference@>=4.5.0 <5.0.0", + "resolved": "https://registry.npmjs.org/lodash.difference/-/lodash.difference-4.5.0.tgz" + }, + "lodash.flatten": { + "version": "4.4.0", + "from": "lodash.flatten@>=4.4.0 <5.0.0", + "resolved": "https://registry.npmjs.org/lodash.flatten/-/lodash.flatten-4.4.0.tgz" + }, + "lodash.foreach": { + "version": "4.5.0", + "from": "lodash.foreach@>=4.5.0 <5.0.0", + "resolved": "https://registry.npmjs.org/lodash.foreach/-/lodash.foreach-4.5.0.tgz" + }, + "lodash.get": { + "version": "4.4.2", + "from": "lodash.get@>=4.4.2 <5.0.0", + "resolved": "https://registry.npmjs.org/lodash.get/-/lodash.get-4.4.2.tgz" + }, + "lodash.isempty": { + "version": "4.4.0", + "from": "lodash.isempty@>=4.4.0 <5.0.0", + "resolved": "https://registry.npmjs.org/lodash.isempty/-/lodash.isempty-4.4.0.tgz" + }, + "lodash.isstring": { + "version": "4.0.1", + "from": "lodash.isstring@>=4.0.1 <5.0.0", + "resolved": "https://registry.npmjs.org/lodash.isstring/-/lodash.isstring-4.0.1.tgz" + }, + "lodash.keys": { + "version": "4.2.0", + "from": "lodash.keys@>=4.2.0 <5.0.0", + "resolved": "https://registry.npmjs.org/lodash.keys/-/lodash.keys-4.2.0.tgz" + }, + "lodash.noop": { + "version": "3.0.1", + "from": "lodash.noop@>=3.0.1 <4.0.0", + "resolved": "https://registry.npmjs.org/lodash.noop/-/lodash.noop-3.0.1.tgz" + }, + "lodash.partial": { + "version": "4.2.1", + "from": "lodash.partial@>=4.2.1 <5.0.0", + "resolved": "https://registry.npmjs.org/lodash.partial/-/lodash.partial-4.2.1.tgz" + }, + "lodash.pick": { + "version": "4.4.0", + "from": "lodash.pick@>=4.4.0 <5.0.0", + "resolved": "https://registry.npmjs.org/lodash.pick/-/lodash.pick-4.4.0.tgz" + }, + "lodash.sample": { + "version": "4.2.1", + "from": "lodash.sample@>=4.2.1 <5.0.0", + "resolved": "https://registry.npmjs.org/lodash.sample/-/lodash.sample-4.2.1.tgz" + }, + "lodash.shuffle": { + "version": "4.2.0", + "from": "lodash.shuffle@>=4.2.0 <5.0.0", + "resolved": "https://registry.npmjs.org/lodash.shuffle/-/lodash.shuffle-4.2.0.tgz" + }, + "lodash.values": { + "version": "4.3.0", + "from": "lodash.values@>=4.3.0 <5.0.0", + "resolved": "https://registry.npmjs.org/lodash.values/-/lodash.values-4.3.0.tgz" + }, + "logger-sharelatex": { + "version": "1.5.6", + "from": "git+https://github.com/sharelatex/logger-sharelatex.git#v1.5.6", + "resolved": "git+https://github.com/sharelatex/logger-sharelatex.git#b2956ec56b582b9f4fc8fdda8dc00c06e77c5537", + "dependencies": { + "assertion-error": { + "version": "1.1.0", + "from": "assertion-error@>=1.0.1 <2.0.0", + "resolved": "https://registry.npmjs.org/assertion-error/-/assertion-error-1.1.0.tgz" + }, + "bunyan": { + "version": "1.5.1", + "from": "bunyan@1.5.1", + "resolved": "https://registry.npmjs.org/bunyan/-/bunyan-1.5.1.tgz" + }, + "chai": { + "version": "4.1.2", + "from": "chai@latest", + "resolved": "https://registry.npmjs.org/chai/-/chai-4.1.2.tgz" + }, + "deep-eql": { + "version": "3.0.1", + "from": "deep-eql@>=3.0.0 <4.0.0", + "resolved": "https://registry.npmjs.org/deep-eql/-/deep-eql-3.0.1.tgz" + }, + "dtrace-provider": { + "version": "0.6.0", + "from": "dtrace-provider@>=0.6.0 <0.7.0", + "resolved": "https://registry.npmjs.org/dtrace-provider/-/dtrace-provider-0.6.0.tgz", + "optional": true + }, + "sandboxed-module": { + "version": "2.0.3", + "from": "sandboxed-module@latest", + "resolved": "https://registry.npmjs.org/sandboxed-module/-/sandboxed-module-2.0.3.tgz" + }, + "sinon": { + "version": "5.0.7", + "from": "sinon@latest", + "resolved": "https://registry.npmjs.org/sinon/-/sinon-5.0.7.tgz", + "dependencies": { + "diff": { + "version": "3.5.0", + "from": "diff@>=3.1.0 <4.0.0", + "resolved": "https://registry.npmjs.org/diff/-/diff-3.5.0.tgz" + }, + "supports-color": { + "version": "5.4.0", + "from": "supports-color@>=5.1.0 <6.0.0", + "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-5.4.0.tgz" + } + } + }, + "timekeeper": { + "version": "1.0.0", + "from": "timekeeper@>=1.0.0 <2.0.0", + "resolved": "https://registry.npmjs.org/timekeeper/-/timekeeper-1.0.0.tgz" + }, + "type-detect": { + "version": "4.0.8", + "from": "type-detect@>=4.0.0 <5.0.0", + "resolved": "https://registry.npmjs.org/type-detect/-/type-detect-4.0.8.tgz" + } + } + }, + "lolex": { + "version": "2.6.0", + "from": "lolex@>=2.2.0 <3.0.0", + "resolved": "https://registry.npmjs.org/lolex/-/lolex-2.6.0.tgz" + }, + "lru-cache": { + "version": "2.7.3", + "from": "lru-cache@>=2.0.0 <3.0.0", + "resolved": "https://registry.npmjs.org/lru-cache/-/lru-cache-2.7.3.tgz" + }, + "lsmod": { + "version": "1.0.0", + "from": "lsmod@1.0.0", + "resolved": "https://registry.npmjs.org/lsmod/-/lsmod-1.0.0.tgz" + }, + "lynx": { + "version": "0.1.1", + "from": "lynx@>=0.1.1 <0.2.0", + "resolved": "https://registry.npmjs.org/lynx/-/lynx-0.1.1.tgz" + }, + "media-typer": { + "version": "0.3.0", + "from": "media-typer@0.3.0", + "resolved": "https://registry.npmjs.org/media-typer/-/media-typer-0.3.0.tgz" + }, + "merge-descriptors": { + "version": "1.0.1", + "from": "merge-descriptors@1.0.1", + "resolved": "https://registry.npmjs.org/merge-descriptors/-/merge-descriptors-1.0.1.tgz" + }, + "mersenne": { + "version": "0.0.4", + "from": "mersenne@>=0.0.3 <0.1.0", + "resolved": "https://registry.npmjs.org/mersenne/-/mersenne-0.0.4.tgz" + }, + "methods": { + "version": "1.1.2", + "from": "methods@>=1.1.2 <1.2.0", + "resolved": "https://registry.npmjs.org/methods/-/methods-1.1.2.tgz" + }, + "metrics-sharelatex": { + "version": "2.0.4", + "from": "git+https://github.com/sharelatex/metrics-sharelatex.git#v2.0.4", + "resolved": "git+https://github.com/sharelatex/metrics-sharelatex.git#49c4bc072f707eb6252adfec540268dfbea1615b", + "dependencies": { + "coffee-script": { + "version": "1.6.0", + "from": "coffee-script@1.6.0", + "resolved": "https://registry.npmjs.org/coffee-script/-/coffee-script-1.6.0.tgz" + }, + "underscore": { + "version": "1.6.0", + "from": "underscore@>=1.6.0 <1.7.0", + "resolved": "https://registry.npmjs.org/underscore/-/underscore-1.6.0.tgz" + } + } + }, + "mime": { + "version": "1.4.1", + "from": "mime@1.4.1", + "resolved": "https://registry.npmjs.org/mime/-/mime-1.4.1.tgz" + }, + "mime-db": { + "version": "1.33.0", + "from": "mime-db@>=1.33.0 <1.34.0", + "resolved": "https://registry.npmjs.org/mime-db/-/mime-db-1.33.0.tgz" + }, + "mime-types": { + "version": "2.1.18", + "from": "mime-types@>=2.1.18 <2.2.0", + "resolved": "https://registry.npmjs.org/mime-types/-/mime-types-2.1.18.tgz" + }, + "minimatch": { + "version": "3.0.4", + "from": "minimatch@2 || 3", + "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-3.0.4.tgz" + }, + "minimist": { + "version": "0.0.8", + "from": "minimist@0.0.8", + "resolved": "https://registry.npmjs.org/minimist/-/minimist-0.0.8.tgz" + }, + "mkdirp": { + "version": "0.5.1", + "from": "mkdirp@~0.5.1", + "resolved": "https://registry.npmjs.org/mkdirp/-/mkdirp-0.5.1.tgz" + }, + "module-details-from-path": { + "version": "1.0.3", + "from": "module-details-from-path@>=1.0.3 <2.0.0", + "resolved": "https://registry.npmjs.org/module-details-from-path/-/module-details-from-path-1.0.3.tgz" + }, + "ms": { + "version": "2.0.0", + "from": "ms@2.0.0", + "resolved": "https://registry.npmjs.org/ms/-/ms-2.0.0.tgz" + }, + "mv": { + "version": "2.1.1", + "from": "mv@~2", + "resolved": "https://registry.npmjs.org/mv/-/mv-2.1.1.tgz", + "optional": true + }, + "nan": { + "version": "2.10.0", + "from": "nan@>=2.0.8 <3.0.0", + "resolved": "https://registry.npmjs.org/nan/-/nan-2.10.0.tgz", + "optional": true + }, + "ncp": { + "version": "2.0.0", + "from": "ncp@~2.0.0", + "resolved": "https://registry.npmjs.org/ncp/-/ncp-2.0.0.tgz", + "optional": true + }, + "negotiator": { + "version": "0.6.1", + "from": "negotiator@0.6.1", + "resolved": "https://registry.npmjs.org/negotiator/-/negotiator-0.6.1.tgz" + }, + "nise": { + "version": "1.3.3", + "from": "nise@>=1.2.0 <2.0.0", + "resolved": "https://registry.npmjs.org/nise/-/nise-1.3.3.tgz", + "dependencies": { + "path-to-regexp": { + "version": "1.7.0", + "from": "path-to-regexp@>=1.7.0 <2.0.0", + "resolved": "https://registry.npmjs.org/path-to-regexp/-/path-to-regexp-1.7.0.tgz" + } + } + }, + "node-fetch": { + "version": "2.3.0", + "from": "node-fetch@>=2.2.0 <3.0.0", + "resolved": "https://registry.npmjs.org/node-fetch/-/node-fetch-2.3.0.tgz" + }, + "node-forge": { + "version": "0.7.6", + "from": "node-forge@>=0.7.4 <0.8.0", + "resolved": "https://registry.npmjs.org/node-forge/-/node-forge-0.7.6.tgz" + }, + "nopt": { + "version": "1.0.10", + "from": "nopt@>=1.0.10 <1.1.0", + "resolved": "https://registry.npmjs.org/nopt/-/nopt-1.0.10.tgz" + }, + "oauth-sign": { + "version": "0.3.0", + "from": "oauth-sign@>=0.3.0 <0.4.0", + "resolved": "https://registry.npmjs.org/oauth-sign/-/oauth-sign-0.3.0.tgz", + "optional": true + }, + "object-keys": { + "version": "1.0.12", + "from": "object-keys@>=1.0.12 <2.0.0", + "resolved": "https://registry.npmjs.org/object-keys/-/object-keys-1.0.12.tgz" + }, + "object.getownpropertydescriptors": { + "version": "2.0.3", + "from": "object.getownpropertydescriptors@>=2.0.3 <3.0.0", + "resolved": "https://registry.npmjs.org/object.getownpropertydescriptors/-/object.getownpropertydescriptors-2.0.3.tgz" + }, + "on-finished": { + "version": "2.3.0", + "from": "on-finished@>=2.3.0 <2.4.0", + "resolved": "https://registry.npmjs.org/on-finished/-/on-finished-2.3.0.tgz" + }, + "on-headers": { + "version": "1.0.1", + "from": "on-headers@>=1.0.1 <1.1.0", + "resolved": "https://registry.npmjs.org/on-headers/-/on-headers-1.0.1.tgz" + }, + "once": { + "version": "1.4.0", + "from": "once@^1.3.0", + "resolved": "https://registry.npmjs.org/once/-/once-1.4.0.tgz" + }, + "options": { + "version": "0.0.6", + "from": "options@>=0.0.5", + "resolved": "https://registry.npmjs.org/options/-/options-0.0.6.tgz" + }, + "p-limit": { + "version": "2.0.0", + "from": "p-limit@>=2.0.0 <3.0.0", + "resolved": "https://registry.npmjs.org/p-limit/-/p-limit-2.0.0.tgz" + }, + "p-try": { + "version": "2.0.0", + "from": "p-try@>=2.0.0 <3.0.0", + "resolved": "https://registry.npmjs.org/p-try/-/p-try-2.0.0.tgz" + }, + "parseurl": { + "version": "1.3.2", + "from": "parseurl@>=1.3.2 <1.4.0", + "resolved": "https://registry.npmjs.org/parseurl/-/parseurl-1.3.2.tgz" + }, + "path-is-absolute": { + "version": "1.0.1", + "from": "path-is-absolute@^1.0.0", + "resolved": "https://registry.npmjs.org/path-is-absolute/-/path-is-absolute-1.0.1.tgz" + }, + "path-parse": { + "version": "1.0.6", + "from": "path-parse@>=1.0.5 <2.0.0", + "resolved": "https://registry.npmjs.org/path-parse/-/path-parse-1.0.6.tgz" + }, + "path-to-regexp": { + "version": "0.1.7", + "from": "path-to-regexp@0.1.7", + "resolved": "https://registry.npmjs.org/path-to-regexp/-/path-to-regexp-0.1.7.tgz" + }, + "pathval": { + "version": "1.1.0", + "from": "pathval@>=1.0.0 <2.0.0", + "resolved": "https://registry.npmjs.org/pathval/-/pathval-1.1.0.tgz" + }, + "pify": { + "version": "4.0.1", + "from": "pify@>=4.0.0 <5.0.0", + "resolved": "https://registry.npmjs.org/pify/-/pify-4.0.1.tgz" + }, + "policyfile": { + "version": "0.0.4", + "from": "policyfile@0.0.4", + "resolved": "https://registry.npmjs.org/policyfile/-/policyfile-0.0.4.tgz" + }, + "process-nextick-args": { + "version": "2.0.0", + "from": "process-nextick-args@>=2.0.0 <2.1.0", + "resolved": "https://registry.npmjs.org/process-nextick-args/-/process-nextick-args-2.0.0.tgz" + }, + "prom-client": { + "version": "11.2.0", + "from": "prom-client@>=11.1.3 <12.0.0", + "resolved": "https://registry.npmjs.org/prom-client/-/prom-client-11.2.0.tgz" + }, + "proxy-addr": { + "version": "2.0.3", + "from": "proxy-addr@>=2.0.3 <2.1.0", + "resolved": "https://registry.npmjs.org/proxy-addr/-/proxy-addr-2.0.3.tgz" + }, + "pseudomap": { + "version": "1.0.2", + "from": "pseudomap@>=1.0.2 <2.0.0", + "resolved": "https://registry.npmjs.org/pseudomap/-/pseudomap-1.0.2.tgz" + }, + "punycode": { + "version": "1.4.1", + "from": "punycode@>=1.4.1 <2.0.0", + "resolved": "https://registry.npmjs.org/punycode/-/punycode-1.4.1.tgz", + "optional": true + }, + "q": { + "version": "0.9.2", + "from": "q@0.9.2", + "resolved": "https://registry.npmjs.org/q/-/q-0.9.2.tgz" + }, + "qs": { + "version": "6.5.2", + "from": "qs@6.5.2", + "resolved": "https://registry.npmjs.org/qs/-/qs-6.5.2.tgz" + }, + "random-bytes": { + "version": "1.0.0", + "from": "random-bytes@>=1.0.0 <1.1.0", + "resolved": "https://registry.npmjs.org/random-bytes/-/random-bytes-1.0.0.tgz" + }, + "range-parser": { + "version": "1.2.0", + "from": "range-parser@>=1.2.0 <1.3.0", + "resolved": "https://registry.npmjs.org/range-parser/-/range-parser-1.2.0.tgz" + }, + "raven": { + "version": "1.2.1", + "from": "raven@>=1.1.3 <2.0.0", + "resolved": "https://registry.npmjs.org/raven/-/raven-1.2.1.tgz" + }, + "raw-body": { + "version": "2.3.3", + "from": "raw-body@2.3.3", + "resolved": "https://registry.npmjs.org/raw-body/-/raw-body-2.3.3.tgz" + }, + "readable-stream": { + "version": "2.3.6", + "from": "readable-stream@>=2.0.0 <3.0.0", + "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-2.3.6.tgz", + "dependencies": { + "isarray": { + "version": "1.0.0", + "from": "isarray@>=1.0.0 <1.1.0", + "resolved": "https://registry.npmjs.org/isarray/-/isarray-1.0.0.tgz" + } + } + }, + "redis": { + "version": "0.12.1", + "from": "redis@>=0.12.1 <0.13.0", + "resolved": "https://registry.npmjs.org/redis/-/redis-0.12.1.tgz" + }, + "redis-commands": { + "version": "1.3.5", + "from": "redis-commands@>=1.2.0 <2.0.0", + "resolved": "https://registry.npmjs.org/redis-commands/-/redis-commands-1.3.5.tgz" + }, + "redis-parser": { + "version": "2.6.0", + "from": "redis-parser@>=2.4.0 <3.0.0", + "resolved": "https://registry.npmjs.org/redis-parser/-/redis-parser-2.6.0.tgz" + }, + "redis-sentinel": { + "version": "0.1.1", + "from": "redis-sentinel@0.1.1", + "resolved": "https://registry.npmjs.org/redis-sentinel/-/redis-sentinel-0.1.1.tgz", + "dependencies": { + "redis": { + "version": "0.11.0", + "from": "redis@>=0.11.0 <0.12.0", + "resolved": "https://registry.npmjs.org/redis/-/redis-0.11.0.tgz" + } + } + }, + "redis-sharelatex": { + "version": "1.0.4", + "from": "git+https://github.com/sharelatex/redis-sharelatex.git#v1.0.4", + "resolved": "git+https://github.com/sharelatex/redis-sharelatex.git#ca4e906559c1405d132e8edd7db763d64a57be62", + "dependencies": { + "async": { + "version": "2.6.1", + "from": "async@>=2.5.0 <3.0.0", + "resolved": "https://registry.npmjs.org/async/-/async-2.6.1.tgz" + }, + "coffee-script": { + "version": "1.8.0", + "from": "coffee-script@1.8.0", + "resolved": "https://registry.npmjs.org/coffee-script/-/coffee-script-1.8.0.tgz" + }, + "lodash": { + "version": "4.17.10", + "from": "lodash@>=4.17.10 <5.0.0", + "resolved": "https://registry.npmjs.org/lodash/-/lodash-4.17.10.tgz" + }, + "mkdirp": { + "version": "0.3.5", + "from": "mkdirp@>=0.3.5 <0.4.0", + "resolved": "https://registry.npmjs.org/mkdirp/-/mkdirp-0.3.5.tgz" + } + } + }, + "request": { + "version": "2.34.0", + "from": "request@>=2.34.0 <2.35.0", + "resolved": "https://registry.npmjs.org/request/-/request-2.34.0.tgz", + "dependencies": { + "mime": { + "version": "1.2.11", + "from": "mime@>=1.2.9 <1.3.0", + "resolved": "https://registry.npmjs.org/mime/-/mime-1.2.11.tgz" + }, + "node-uuid": { + "version": "1.4.8", + "from": "node-uuid@>=1.4.0 <1.5.0", + "resolved": "https://registry.npmjs.org/node-uuid/-/node-uuid-1.4.8.tgz" + }, + "qs": { + "version": "0.6.6", + "from": "qs@>=0.6.0 <0.7.0", + "resolved": "https://registry.npmjs.org/qs/-/qs-0.6.6.tgz" + } + } + }, + "require-in-the-middle": { + "version": "3.1.0", + "from": "require-in-the-middle@>=3.0.0 <4.0.0", + "resolved": "https://registry.npmjs.org/require-in-the-middle/-/require-in-the-middle-3.1.0.tgz" + }, + "require-like": { + "version": "0.1.2", + "from": "require-like@0.1.2", + "resolved": "https://registry.npmjs.org/require-like/-/require-like-0.1.2.tgz" + }, + "resolve": { + "version": "1.8.1", + "from": "resolve@>=1.5.0 <2.0.0", + "resolved": "https://registry.npmjs.org/resolve/-/resolve-1.8.1.tgz" + }, + "retry-axios": { + "version": "0.3.2", + "from": "retry-axios@>=0.3.2 <0.4.0", + "resolved": "https://registry.npmjs.org/retry-axios/-/retry-axios-0.3.2.tgz" + }, + "retry-request": { + "version": "4.0.0", + "from": "retry-request@>=4.0.0 <5.0.0", + "resolved": "https://registry.npmjs.org/retry-request/-/retry-request-4.0.0.tgz" + }, + "rimraf": { + "version": "2.4.5", + "from": "rimraf@~2.4.0", + "resolved": "https://registry.npmjs.org/rimraf/-/rimraf-2.4.5.tgz" + }, + "safe-buffer": { + "version": "5.1.1", + "from": "safe-buffer@5.1.1", + "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.1.1.tgz" + }, + "safe-json-stringify": { + "version": "1.1.0", + "from": "safe-json-stringify@>=1.0.0 <2.0.0", + "resolved": "https://registry.npmjs.org/safe-json-stringify/-/safe-json-stringify-1.1.0.tgz", + "optional": true + }, + "safer-buffer": { + "version": "2.1.2", + "from": "safer-buffer@>=2.1.2 <3.0.0", + "resolved": "https://registry.npmjs.org/safer-buffer/-/safer-buffer-2.1.2.tgz" + }, + "samsam": { + "version": "1.3.0", + "from": "samsam@1.3.0", + "resolved": "https://registry.npmjs.org/samsam/-/samsam-1.3.0.tgz" + }, + "semver": { + "version": "5.6.0", + "from": "semver@>=5.5.0 <6.0.0", + "resolved": "https://registry.npmjs.org/semver/-/semver-5.6.0.tgz" + }, + "send": { + "version": "0.16.2", + "from": "send@0.16.2", + "resolved": "https://registry.npmjs.org/send/-/send-0.16.2.tgz", + "dependencies": { + "statuses": { + "version": "1.4.0", + "from": "statuses@>=1.4.0 <1.5.0", + "resolved": "https://registry.npmjs.org/statuses/-/statuses-1.4.0.tgz" + } + } + }, + "serve-static": { + "version": "1.13.2", + "from": "serve-static@1.13.2", + "resolved": "https://registry.npmjs.org/serve-static/-/serve-static-1.13.2.tgz" + }, + "session.socket.io": { + "version": "0.1.6", + "from": "session.socket.io@>=0.1.6 <0.2.0", + "resolved": "https://registry.npmjs.org/session.socket.io/-/session.socket.io-0.1.6.tgz" + }, + "setprototypeof": { + "version": "1.1.0", + "from": "setprototypeof@1.1.0", + "resolved": "https://registry.npmjs.org/setprototypeof/-/setprototypeof-1.1.0.tgz" + }, + "settings-sharelatex": { + "version": "1.0.0", + "from": "git+https://github.com/sharelatex/settings-sharelatex.git#v1.0.0", + "resolved": "git+https://github.com/sharelatex/settings-sharelatex.git#cbc5e41c1dbe6789721a14b3fdae05bf22546559", + "dependencies": { + "coffee-script": { + "version": "1.6.0", + "from": "coffee-script@1.6.0", + "resolved": "https://registry.npmjs.org/coffee-script/-/coffee-script-1.6.0.tgz" + } + } + }, + "shimmer": { + "version": "1.2.0", + "from": "shimmer@>=1.2.0 <2.0.0", + "resolved": "https://registry.npmjs.org/shimmer/-/shimmer-1.2.0.tgz" + }, + "sigmund": { + "version": "1.0.1", + "from": "sigmund@>=1.0.0 <1.1.0", + "resolved": "https://registry.npmjs.org/sigmund/-/sigmund-1.0.1.tgz" + }, + "sntp": { + "version": "0.2.4", + "from": "sntp@>=0.2.0 <0.3.0", + "resolved": "https://registry.npmjs.org/sntp/-/sntp-0.2.4.tgz", + "optional": true + }, + "socket.io": { + "version": "0.9.16", + "from": "socket.io@0.9.16", + "resolved": "https://registry.npmjs.org/socket.io/-/socket.io-0.9.16.tgz", + "dependencies": { + "redis": { + "version": "0.7.3", + "from": "redis@0.7.3", + "resolved": "https://registry.npmjs.org/redis/-/redis-0.7.3.tgz", + "optional": true + }, + "socket.io-client": { + "version": "0.9.16", + "from": "socket.io-client@0.9.16", + "resolved": "https://registry.npmjs.org/socket.io-client/-/socket.io-client-0.9.16.tgz" + } + } + }, + "socket.io-client": { + "version": "0.9.17", + "from": "socket.io-client@>=0.9.16 <0.10.0", + "resolved": "https://registry.npmjs.org/socket.io-client/-/socket.io-client-0.9.17.tgz" + }, + "source-map": { + "version": "0.6.1", + "from": "source-map@>=0.6.1 <0.7.0", + "resolved": "https://registry.npmjs.org/source-map/-/source-map-0.6.1.tgz" + }, + "split": { + "version": "1.0.1", + "from": "split@>=1.0.0 <2.0.0", + "resolved": "https://registry.npmjs.org/split/-/split-1.0.1.tgz" + }, + "stack-trace": { + "version": "0.0.9", + "from": "stack-trace@0.0.9", + "resolved": "https://registry.npmjs.org/stack-trace/-/stack-trace-0.0.9.tgz" + }, + "statsd-parser": { + "version": "0.0.4", + "from": "statsd-parser@>=0.0.4 <0.1.0", + "resolved": "https://registry.npmjs.org/statsd-parser/-/statsd-parser-0.0.4.tgz" + }, + "statuses": { + "version": "1.5.0", + "from": "statuses@>=1.4.0 <2.0.0", + "resolved": "https://registry.npmjs.org/statuses/-/statuses-1.5.0.tgz" + }, + "stream-shift": { + "version": "1.0.0", + "from": "stream-shift@>=1.0.0 <2.0.0", + "resolved": "https://registry.npmjs.org/stream-shift/-/stream-shift-1.0.0.tgz" + }, + "string_decoder": { + "version": "1.1.1", + "from": "string_decoder@>=1.1.1 <1.2.0", + "resolved": "https://registry.npmjs.org/string_decoder/-/string_decoder-1.1.1.tgz" + }, + "strip-ansi": { + "version": "0.3.0", + "from": "strip-ansi@>=0.3.0 <0.4.0", + "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-0.3.0.tgz" + }, + "supports-color": { + "version": "0.2.0", + "from": "supports-color@>=0.2.0 <0.3.0", + "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-0.2.0.tgz" + }, + "tdigest": { + "version": "0.1.1", + "from": "tdigest@>=0.1.1 <0.2.0", + "resolved": "https://registry.npmjs.org/tdigest/-/tdigest-0.1.1.tgz" + }, + "teeny-request": { + "version": "3.11.3", + "from": "teeny-request@>=3.6.0 <4.0.0", + "resolved": "https://registry.npmjs.org/teeny-request/-/teeny-request-3.11.3.tgz", + "dependencies": { + "uuid": { + "version": "3.3.2", + "from": "uuid@>=3.3.2 <4.0.0", + "resolved": "https://registry.npmjs.org/uuid/-/uuid-3.3.2.tgz" + } + } + }, + "text-encoding": { + "version": "0.6.4", + "from": "text-encoding@>=0.6.4 <0.7.0", + "resolved": "https://registry.npmjs.org/text-encoding/-/text-encoding-0.6.4.tgz" + }, + "through": { + "version": "2.3.8", + "from": "through@>=2.0.0 <3.0.0", + "resolved": "https://registry.npmjs.org/through/-/through-2.3.8.tgz" + }, + "through2": { + "version": "2.0.5", + "from": "through2@>=2.0.3 <3.0.0", + "resolved": "https://registry.npmjs.org/through2/-/through2-2.0.5.tgz" + }, + "tinycolor": { + "version": "0.0.1", + "from": "tinycolor@>=0.0.0 <1.0.0", + "resolved": "https://registry.npmjs.org/tinycolor/-/tinycolor-0.0.1.tgz" + }, + "tough-cookie": { + "version": "2.3.4", + "from": "tough-cookie@>=0.12.0", + "resolved": "https://registry.npmjs.org/tough-cookie/-/tough-cookie-2.3.4.tgz", + "optional": true + }, + "tunnel-agent": { + "version": "0.3.0", + "from": "tunnel-agent@>=0.3.0 <0.4.0", + "resolved": "https://registry.npmjs.org/tunnel-agent/-/tunnel-agent-0.3.0.tgz", + "optional": true + }, + "type-is": { + "version": "1.6.16", + "from": "type-is@>=1.6.16 <1.7.0", + "resolved": "https://registry.npmjs.org/type-is/-/type-is-1.6.16.tgz" + }, + "uglify-js": { + "version": "1.2.5", + "from": "uglify-js@1.2.5", + "resolved": "https://registry.npmjs.org/uglify-js/-/uglify-js-1.2.5.tgz" + }, + "underscore": { + "version": "1.7.0", + "from": "underscore@>=1.7.0 <1.8.0", + "resolved": "https://registry.npmjs.org/underscore/-/underscore-1.7.0.tgz" + }, + "underscore.string": { + "version": "2.2.1", + "from": "underscore.string@>=2.2.1 <2.3.0", + "resolved": "https://registry.npmjs.org/underscore.string/-/underscore.string-2.2.1.tgz" + }, + "unpipe": { + "version": "1.0.0", + "from": "unpipe@1.0.0", + "resolved": "https://registry.npmjs.org/unpipe/-/unpipe-1.0.0.tgz" + }, + "util-deprecate": { + "version": "1.0.2", + "from": "util-deprecate@>=1.0.1 <1.1.0", + "resolved": "https://registry.npmjs.org/util-deprecate/-/util-deprecate-1.0.2.tgz" + }, + "util.promisify": { + "version": "1.0.0", + "from": "util.promisify@>=1.0.0 <2.0.0", + "resolved": "https://registry.npmjs.org/util.promisify/-/util.promisify-1.0.0.tgz" + }, + "utils-merge": { + "version": "1.0.1", + "from": "utils-merge@1.0.1", + "resolved": "https://registry.npmjs.org/utils-merge/-/utils-merge-1.0.1.tgz" + }, + "uuid": { + "version": "3.0.0", + "from": "uuid@3.0.0", + "resolved": "https://registry.npmjs.org/uuid/-/uuid-3.0.0.tgz" + }, + "vary": { + "version": "1.1.2", + "from": "vary@>=1.1.2 <1.2.0", + "resolved": "https://registry.npmjs.org/vary/-/vary-1.1.2.tgz" + }, + "which": { + "version": "1.0.9", + "from": "which@>=1.0.5 <1.1.0", + "resolved": "https://registry.npmjs.org/which/-/which-1.0.9.tgz" + }, + "wrappy": { + "version": "1.0.2", + "from": "wrappy@1", + "resolved": "https://registry.npmjs.org/wrappy/-/wrappy-1.0.2.tgz" + }, + "ws": { + "version": "0.4.32", + "from": "ws@>=0.4.0 <0.5.0", + "resolved": "https://registry.npmjs.org/ws/-/ws-0.4.32.tgz", + "dependencies": { + "commander": { + "version": "2.1.0", + "from": "commander@>=2.1.0 <2.2.0", + "resolved": "https://registry.npmjs.org/commander/-/commander-2.1.0.tgz" + }, + "nan": { + "version": "1.0.0", + "from": "nan@>=1.0.0 <1.1.0", + "resolved": "https://registry.npmjs.org/nan/-/nan-1.0.0.tgz" + } + } + }, + "xmlhttprequest": { + "version": "1.4.2", + "from": "xmlhttprequest@1.4.2", + "resolved": "https://registry.npmjs.org/xmlhttprequest/-/xmlhttprequest-1.4.2.tgz" + }, + "xtend": { + "version": "4.0.1", + "from": "xtend@>=4.0.1 <4.1.0", + "resolved": "https://registry.npmjs.org/xtend/-/xtend-4.0.1.tgz" + }, + "yallist": { + "version": "2.1.2", + "from": "yallist@>=2.1.2 <3.0.0", + "resolved": "https://registry.npmjs.org/yallist/-/yallist-2.1.2.tgz" + }, + "zeparser": { + "version": "0.0.5", + "from": "zeparser@0.0.5", + "resolved": "https://registry.npmjs.org/zeparser/-/zeparser-0.0.5.tgz" + } + } +} diff --git a/services/real-time/package.json b/services/real-time/package.json index 6e4f378c32..a2466ea67f 100644 --- a/services/real-time/package.json +++ b/services/real-time/package.json @@ -29,7 +29,7 @@ "express": "^4.10.1", "express-session": "^1.9.1", "logger-sharelatex": "git+https://github.com/sharelatex/logger-sharelatex.git#v1.5.6", - "metrics-sharelatex": "git+https://github.com/sharelatex/metrics-sharelatex.git#v1.4.0", + "metrics-sharelatex": "git+https://github.com/sharelatex/metrics-sharelatex.git#v2.0.4", "redis-sharelatex": "git+https://github.com/sharelatex/redis-sharelatex.git#v1.0.4", "request": "~2.34.0", "session.socket.io": "^0.1.6", @@ -41,14 +41,6 @@ "bunyan": "~0.22.3", "chai": "~1.9.1", "cookie-signature": "^1.0.5", - "grunt": "~0.4.4", - "grunt-bunyan": "~0.5.0", - "grunt-contrib-clean": "~0.5.0", - "grunt-contrib-coffee": "~0.10.1", - "grunt-execute": "~0.2.1", - "grunt-forever": "~0.4.4", - "grunt-mocha-test": "~0.10.2", - "grunt-shell": "~0.7.0", "sandboxed-module": "~0.3.0", "sinon": "~1.5.2", "mocha": "^4.0.1", From 64f3d32c6c7cf4e8da47c5ae516677580f75e216 Mon Sep 17 00:00:00 2001 From: Henry Oswald Date: Tue, 4 Dec 2018 13:22:09 +0000 Subject: [PATCH 135/491] log out status calls --- services/real-time/app.coffee | 1 + 1 file changed, 1 insertion(+) diff --git a/services/real-time/app.coffee b/services/real-time/app.coffee index b66375addb..37a45a957b 100644 --- a/services/real-time/app.coffee +++ b/services/real-time/app.coffee @@ -49,6 +49,7 @@ app.get "/", (req, res, next) -> res.send "real-time-sharelatex is alive" app.get "/status", (req, res, next) -> + console.log("got status") res.send "real-time-sharelatex is alive" rclient = require("redis-sharelatex").createClient(Settings.redis.realtime) From 2282518c90fe8d1c1b6e3b95420834d8a76ac31c Mon Sep 17 00:00:00 2001 From: Henry Oswald Date: Tue, 4 Dec 2018 13:26:47 +0000 Subject: [PATCH 136/491] print out req.query --- services/real-time/app.coffee | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/services/real-time/app.coffee b/services/real-time/app.coffee index 37a45a957b..9c89bc5cc8 100644 --- a/services/real-time/app.coffee +++ b/services/real-time/app.coffee @@ -49,7 +49,7 @@ app.get "/", (req, res, next) -> res.send "real-time-sharelatex is alive" app.get "/status", (req, res, next) -> - console.log("got status") + console.log("got status", req.query) res.send "real-time-sharelatex is alive" rclient = require("redis-sharelatex").createClient(Settings.redis.realtime) From fc2d2405f4ba99753e0b611b16d1e2f25b637c79 Mon Sep 17 00:00:00 2001 From: Henry Oswald Date: Tue, 4 Dec 2018 13:32:37 +0000 Subject: [PATCH 137/491] log out io clients --- services/real-time/app.coffee | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/services/real-time/app.coffee b/services/real-time/app.coffee index 9c89bc5cc8..0857fcc835 100644 --- a/services/real-time/app.coffee +++ b/services/real-time/app.coffee @@ -27,8 +27,7 @@ io = require('socket.io').listen(server) # Bind to sessions sessionStore = new RedisStore(client: sessionRedisClient) cookieParser = CookieParser(Settings.security.sessionSecret) -console.log(Settings.security.sessionSecret, Settings.cookieName) -console.log(Settings) + sessionSockets = new SessionSockets(io, sessionStore, cookieParser, Settings.cookieName) io.configure -> @@ -49,7 +48,7 @@ app.get "/", (req, res, next) -> res.send "real-time-sharelatex is alive" app.get "/status", (req, res, next) -> - console.log("got status", req.query) + console.log("got status", req.query, io.sockets.clients()?.length) res.send "real-time-sharelatex is alive" rclient = require("redis-sharelatex").createClient(Settings.redis.realtime) From 9a851d6ccebcab6ab6a7d6cccc4134f93a07f427 Mon Sep 17 00:00:00 2001 From: Henry Oswald Date: Tue, 4 Dec 2018 13:47:04 +0000 Subject: [PATCH 138/491] mvp for safe shutdown --- services/real-time/app.coffee | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/services/real-time/app.coffee b/services/real-time/app.coffee index 0857fcc835..034dc2d915 100644 --- a/services/real-time/app.coffee +++ b/services/real-time/app.coffee @@ -78,3 +78,20 @@ server.listen port, host, (error) -> # Stop huge stack traces in logs from all the socket.io parsing steps. Error.stackTraceLimit = 10 + + +shutdownCleanly = (signal) -> + return () -> + logger.log signal: signal, "received interrupt, cleaning up" + connectedClients = io.sockets.clients()?.length + logger.log {connectedClients, signal}, "looking to shut down process" + if connectedClients == 0 + logger.log("no clients connected, exiting") + process.exit() + else + setTimeout () -> + shutdownCleanly(signal) + , 10000 + +for signal in ['SIGINT', 'SIGHUP', 'SIGQUIT', 'SIGUSR1', 'SIGUSR2', 'SIGTERM', 'SIGABRT'] + process.on signal, shutdownCleanly(signal) From 7f2decae4d4e7b33787693729516898548c5bc52 Mon Sep 17 00:00:00 2001 From: Henry Oswald Date: Tue, 4 Dec 2018 14:06:39 +0000 Subject: [PATCH 139/491] start drain after 3 hours --- services/real-time/app.coffee | 39 ++++++++++++------- .../app/coffee/ConnectedUsersManager.coffee | 2 - .../coffee/DocumentUpdaterController.coffee | 2 - .../real-time/config/settings.defaults.coffee | 4 +- 4 files changed, 29 insertions(+), 18 deletions(-) diff --git a/services/real-time/app.coffee b/services/real-time/app.coffee index 034dc2d915..20c89a2681 100644 --- a/services/real-time/app.coffee +++ b/services/real-time/app.coffee @@ -19,6 +19,8 @@ Metrics.initialize(Settings.appName or "real-time") Metrics.event_loop.monitor(logger) +DrainManager = require("./app/js/DrainManager") + # Set up socket.io server app = express() server = require('http').createServer(app) @@ -81,17 +83,28 @@ Error.stackTraceLimit = 10 shutdownCleanly = (signal) -> - return () -> - logger.log signal: signal, "received interrupt, cleaning up" - connectedClients = io.sockets.clients()?.length - logger.log {connectedClients, signal}, "looking to shut down process" - if connectedClients == 0 - logger.log("no clients connected, exiting") - process.exit() - else - setTimeout () -> - shutdownCleanly(signal) - , 10000 + connectedClients = io.sockets.clients()?.length + logger.log {connectedClients, signal}, "looking to shut down process" + if connectedClients == 0 + logger.log("no clients connected, exiting") + process.exit() + else + setTimeout () -> + shutdownCleanly(signal) + , 10000 -for signal in ['SIGINT', 'SIGHUP', 'SIGQUIT', 'SIGUSR1', 'SIGUSR2', 'SIGTERM', 'SIGABRT'] - process.on signal, shutdownCleanly(signal) +forceDrain = -> + THREE_HOURS = 60 * 1000 * 60 * 3 + setTimeout( -> , + logger.log({delay_ms:THREE_HOURS}, "starting drain") + DrainManager.startDrain(io, 4) + , THREE_HOURS) + + +if Settings.drainBeforeShutdown + logger.log "drainBeforeShutdown enabled" + for signal in ['SIGINT', 'SIGHUP', 'SIGQUIT', 'SIGUSR1', 'SIGUSR2', 'SIGTERM', 'SIGABRT'] + logger.log signal: signal, "received interrupt, cleaning up" + process.on signal, -> + shutdownCleanly(signal) + forceDrain() diff --git a/services/real-time/app/coffee/ConnectedUsersManager.coffee b/services/real-time/app/coffee/ConnectedUsersManager.coffee index 38370da464..831a237356 100644 --- a/services/real-time/app/coffee/ConnectedUsersManager.coffee +++ b/services/real-time/app/coffee/ConnectedUsersManager.coffee @@ -5,8 +5,6 @@ redis = require("redis-sharelatex") rclient = redis.createClient(Settings.redis.realtime) Keys = Settings.redis.realtime.key_schema -console.log Settings.redis.realtime, "REALTIME" - ONE_HOUR_IN_S = 60 * 60 ONE_DAY_IN_S = ONE_HOUR_IN_S * 24 FOUR_DAYS_IN_S = ONE_DAY_IN_S * 4 diff --git a/services/real-time/app/coffee/DocumentUpdaterController.coffee b/services/real-time/app/coffee/DocumentUpdaterController.coffee index 05a70fe3a7..451dc812bc 100644 --- a/services/real-time/app/coffee/DocumentUpdaterController.coffee +++ b/services/real-time/app/coffee/DocumentUpdaterController.coffee @@ -4,8 +4,6 @@ redis = require("redis-sharelatex") rclient = redis.createClient(settings.redis.documentupdater) SafeJsonParse = require "./SafeJsonParse" -console.log "REDIS", settings.redis - MESSAGE_SIZE_LOG_LIMIT = 1024 * 1024 # 1Mb module.exports = DocumentUpdaterController = diff --git a/services/real-time/config/settings.defaults.coffee b/services/real-time/config/settings.defaults.coffee index 485789cb12..270c32aa77 100644 --- a/services/real-time/config/settings.defaults.coffee +++ b/services/real-time/config/settings.defaults.coffee @@ -40,4 +40,6 @@ module.exports = cookieName: process.env['COOKIE_NAME'] or "sharelatex.sid" - max_doc_length: 2 * 1024 * 1024 # 2mb \ No newline at end of file + max_doc_length: 2 * 1024 * 1024 # 2mb + + drainBeforeShutdown: process.env['DRAIN_BEFORE_SHUTDOWN'] or false \ No newline at end of file From 8073cdea75af81bc29bef5f084525ab1e33976f5 Mon Sep 17 00:00:00 2001 From: Henry Oswald Date: Tue, 4 Dec 2018 14:11:28 +0000 Subject: [PATCH 140/491] improve logging --- services/real-time/app.coffee | 17 ++++++++++------- .../app/coffee/ConnectedUsersManager.coffee | 1 - 2 files changed, 10 insertions(+), 8 deletions(-) diff --git a/services/real-time/app.coffee b/services/real-time/app.coffee index 20c89a2681..275667d477 100644 --- a/services/real-time/app.coffee +++ b/services/real-time/app.coffee @@ -1,5 +1,9 @@ +Metrics = require("metrics-sharelatex") +Metrics.initialize(Settings.appName or "real-time") + logger = require "logger-sharelatex" logger.initialize("real-time-sharelatex") +Metrics.event_loop.monitor(logger) express = require("express") session = require("express-session") @@ -14,9 +18,7 @@ RedisStore = require('connect-redis')(session) SessionSockets = require('session.socket.io') CookieParser = require("cookie-parser") -Metrics = require("metrics-sharelatex") -Metrics.initialize(Settings.appName or "real-time") -Metrics.event_loop.monitor(logger) + DrainManager = require("./app/js/DrainManager") @@ -84,21 +86,22 @@ Error.stackTraceLimit = 10 shutdownCleanly = (signal) -> connectedClients = io.sockets.clients()?.length - logger.log {connectedClients, signal}, "looking to shut down process" if connectedClients == 0 logger.log("no clients connected, exiting") process.exit() else + logger.log {connectedClients}, "clients still connected, not shutting down yet" setTimeout () -> shutdownCleanly(signal) , 10000 forceDrain = -> THREE_HOURS = 60 * 1000 * 60 * 3 - setTimeout( -> , - logger.log({delay_ms:THREE_HOURS}, "starting drain") + logger.log {delay_ms:THREE_HOURS}, "starting force drain after timeout" + setTimeout ()-> + logger.log "starting drain" DrainManager.startDrain(io, 4) - , THREE_HOURS) + , THREE_HOURS if Settings.drainBeforeShutdown diff --git a/services/real-time/app/coffee/ConnectedUsersManager.coffee b/services/real-time/app/coffee/ConnectedUsersManager.coffee index 831a237356..fe7d7efb85 100644 --- a/services/real-time/app/coffee/ConnectedUsersManager.coffee +++ b/services/real-time/app/coffee/ConnectedUsersManager.coffee @@ -12,7 +12,6 @@ FOUR_DAYS_IN_S = ONE_DAY_IN_S * 4 USER_TIMEOUT_IN_S = ONE_HOUR_IN_S / 4 module.exports = - # Use the same method for when a user connects, and when a user sends a cursor # update. This way we don't care if the connected_user key has expired when # we receive a cursor update. From 2418e5db574505143c50e1993b116b3b22831e26 Mon Sep 17 00:00:00 2001 From: Henry Oswald Date: Tue, 4 Dec 2018 14:21:05 +0000 Subject: [PATCH 141/491] use delayExitUntilDrained --- services/real-time/app.coffee | 4 ++-- services/real-time/config/settings.defaults.coffee | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/services/real-time/app.coffee b/services/real-time/app.coffee index 275667d477..d3ee470a8e 100644 --- a/services/real-time/app.coffee +++ b/services/real-time/app.coffee @@ -1,4 +1,5 @@ Metrics = require("metrics-sharelatex") +Settings = require "settings-sharelatex" Metrics.initialize(Settings.appName or "real-time") logger = require "logger-sharelatex" @@ -8,7 +9,6 @@ Metrics.event_loop.monitor(logger) express = require("express") session = require("express-session") redis = require("redis-sharelatex") -Settings = require "settings-sharelatex" if Settings.sentry?.dsn? logger.initializeErrorReporting(Settings.sentry.dsn) @@ -104,7 +104,7 @@ forceDrain = -> , THREE_HOURS -if Settings.drainBeforeShutdown +if Settings.delayExitUntilDrained logger.log "drainBeforeShutdown enabled" for signal in ['SIGINT', 'SIGHUP', 'SIGQUIT', 'SIGUSR1', 'SIGUSR2', 'SIGTERM', 'SIGABRT'] logger.log signal: signal, "received interrupt, cleaning up" diff --git a/services/real-time/config/settings.defaults.coffee b/services/real-time/config/settings.defaults.coffee index 270c32aa77..75f86db306 100644 --- a/services/real-time/config/settings.defaults.coffee +++ b/services/real-time/config/settings.defaults.coffee @@ -42,4 +42,4 @@ module.exports = max_doc_length: 2 * 1024 * 1024 # 2mb - drainBeforeShutdown: process.env['DRAIN_BEFORE_SHUTDOWN'] or false \ No newline at end of file + delayExitUntilDrained: process.env['DELAY_EXIT_UNTIL_DRAINED'] or false \ No newline at end of file From 05611de15e13775c9890a3a0c6e2a1c31f69f8bb Mon Sep 17 00:00:00 2001 From: Henry Oswald Date: Tue, 4 Dec 2018 14:31:07 +0000 Subject: [PATCH 142/491] use FORCE_DRAIN_MS_DELAY --- services/real-time/app.coffee | 9 ++++----- services/real-time/config/settings.defaults.coffee | 2 +- 2 files changed, 5 insertions(+), 6 deletions(-) diff --git a/services/real-time/app.coffee b/services/real-time/app.coffee index d3ee470a8e..17d2d5bcf9 100644 --- a/services/real-time/app.coffee +++ b/services/real-time/app.coffee @@ -96,15 +96,14 @@ shutdownCleanly = (signal) -> , 10000 forceDrain = -> - THREE_HOURS = 60 * 1000 * 60 * 3 - logger.log {delay_ms:THREE_HOURS}, "starting force drain after timeout" + forceDrainMsDelay = parseInt(Settings.forceDrainMsDelay, 10) + logger.log {delay_ms:forceDrainMsDelay}, "starting force drain after timeout" setTimeout ()-> logger.log "starting drain" DrainManager.startDrain(io, 4) - , THREE_HOURS + , Settings.forceDrainMsDelay - -if Settings.delayExitUntilDrained +if Settings.forceDrainMsDelay? logger.log "drainBeforeShutdown enabled" for signal in ['SIGINT', 'SIGHUP', 'SIGQUIT', 'SIGUSR1', 'SIGUSR2', 'SIGTERM', 'SIGABRT'] logger.log signal: signal, "received interrupt, cleaning up" diff --git a/services/real-time/config/settings.defaults.coffee b/services/real-time/config/settings.defaults.coffee index 75f86db306..8eac78da5a 100644 --- a/services/real-time/config/settings.defaults.coffee +++ b/services/real-time/config/settings.defaults.coffee @@ -42,4 +42,4 @@ module.exports = max_doc_length: 2 * 1024 * 1024 # 2mb - delayExitUntilDrained: process.env['DELAY_EXIT_UNTIL_DRAINED'] or false \ No newline at end of file + forceDrainMsDelay: process.env['FORCE_DRAIN_MS_DELAY'] or false \ No newline at end of file From b834049eeb65cb01bd9d0e17cf5037ce971efa1b Mon Sep 17 00:00:00 2001 From: Henry Oswald Date: Tue, 4 Dec 2018 14:33:25 +0000 Subject: [PATCH 143/491] improve delay ms logging --- services/real-time/app.coffee | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/services/real-time/app.coffee b/services/real-time/app.coffee index 17d2d5bcf9..8d4f8ecac7 100644 --- a/services/real-time/app.coffee +++ b/services/real-time/app.coffee @@ -96,17 +96,17 @@ shutdownCleanly = (signal) -> , 10000 forceDrain = -> - forceDrainMsDelay = parseInt(Settings.forceDrainMsDelay, 10) - logger.log {delay_ms:forceDrainMsDelay}, "starting force drain after timeout" + logger.log {delay_ms:Settings.forceDrainMsDelay}, "starting force drain after timeout" setTimeout ()-> logger.log "starting drain" DrainManager.startDrain(io, 4) , Settings.forceDrainMsDelay - + if Settings.forceDrainMsDelay? - logger.log "drainBeforeShutdown enabled" + Settings.forceDrainMsDelay = parseInt(Settings.forceDrainMsDelay, 10) + logger.log forceDrainMsDelay: Settings.forceDrainMsDelay,"forceDrainMsDelay enabled" for signal in ['SIGINT', 'SIGHUP', 'SIGQUIT', 'SIGUSR1', 'SIGUSR2', 'SIGTERM', 'SIGABRT'] - logger.log signal: signal, "received interrupt, cleaning up" process.on signal, -> + logger.log signal: signal, "received interrupt, cleaning up" shutdownCleanly(signal) forceDrain() From 258617fbd4d85bb290b658d167ec9a2d122f61d2 Mon Sep 17 00:00:00 2001 From: Henry Oswald Date: Tue, 4 Dec 2018 15:58:42 +0000 Subject: [PATCH 144/491] bump metrics --- services/real-time/npm-shrinkwrap.json | 167 ++++++++++++++++++++++++- services/real-time/package.json | 2 +- 2 files changed, 165 insertions(+), 4 deletions(-) diff --git a/services/real-time/npm-shrinkwrap.json b/services/real-time/npm-shrinkwrap.json index 1da4679f14..8050f90cc7 100644 --- a/services/real-time/npm-shrinkwrap.json +++ b/services/real-time/npm-shrinkwrap.json @@ -177,6 +177,12 @@ "resolved": "https://registry.npmjs.org/assert-plus/-/assert-plus-0.1.5.tgz", "optional": true }, + "assertion-error": { + "version": "1.0.0", + "from": "assertion-error@1.0.0", + "resolved": "https://registry.npmjs.org/assertion-error/-/assertion-error-1.0.0.tgz", + "dev": true + }, "async": { "version": "0.9.2", "from": "async@>=0.9.0 <0.10.0", @@ -203,6 +209,12 @@ "from": "balanced-match@^1.0.0", "resolved": "https://registry.npmjs.org/balanced-match/-/balanced-match-1.0.0.tgz" }, + "base64-url": { + "version": "1.2.1", + "from": "base64-url@1.2.1", + "resolved": "https://registry.npmjs.org/base64-url/-/base64-url-1.2.1.tgz", + "dev": true + }, "base64id": { "version": "0.1.0", "from": "base64id@0.1.0", @@ -243,6 +255,12 @@ "from": "brace-expansion@^1.1.7", "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-1.1.11.tgz" }, + "browser-stdout": { + "version": "1.3.0", + "from": "browser-stdout@1.3.0", + "resolved": "https://registry.npmjs.org/browser-stdout/-/browser-stdout-1.3.0.tgz", + "dev": true + }, "buffer-equal-constant-time": { "version": "1.0.1", "from": "buffer-equal-constant-time@1.0.1", @@ -253,11 +271,35 @@ "from": "builtin-modules@>=3.0.0 <4.0.0", "resolved": "https://registry.npmjs.org/builtin-modules/-/builtin-modules-3.0.0.tgz" }, + "bunyan": { + "version": "0.22.3", + "from": "bunyan@>=0.22.3 <0.23.0", + "resolved": "https://registry.npmjs.org/bunyan/-/bunyan-0.22.3.tgz", + "dev": true + }, + "buster-core": { + "version": "0.6.4", + "from": "buster-core@0.6.4", + "resolved": "https://registry.npmjs.org/buster-core/-/buster-core-0.6.4.tgz", + "dev": true + }, + "buster-format": { + "version": "0.5.6", + "from": "buster-format@>=0.5.0 <0.6.0", + "resolved": "https://registry.npmjs.org/buster-format/-/buster-format-0.5.6.tgz", + "dev": true + }, "bytes": { "version": "3.0.0", "from": "bytes@3.0.0", "resolved": "https://registry.npmjs.org/bytes/-/bytes-3.0.0.tgz" }, + "chai": { + "version": "1.9.2", + "from": "chai@>=1.9.1 <1.10.0", + "resolved": "https://registry.npmjs.org/chai/-/chai-1.9.2.tgz", + "dev": true + }, "chalk": { "version": "0.5.1", "from": "chalk@>=0.5.0 <0.6.0", @@ -378,6 +420,12 @@ "from": "debug@2.6.9", "resolved": "https://registry.npmjs.org/debug/-/debug-2.6.9.tgz" }, + "deep-eql": { + "version": "0.1.3", + "from": "deep-eql@0.1.3", + "resolved": "https://registry.npmjs.org/deep-eql/-/deep-eql-0.1.3.tgz", + "dev": true + }, "define-properties": { "version": "1.1.3", "from": "define-properties@>=1.1.2 <2.0.0", @@ -409,6 +457,13 @@ "from": "diff@1.0.7", "resolved": "https://registry.npmjs.org/diff/-/diff-1.0.7.tgz" }, + "dtrace-provider": { + "version": "0.2.8", + "from": "dtrace-provider@0.2.8", + "resolved": "https://registry.npmjs.org/dtrace-provider/-/dtrace-provider-0.2.8.tgz", + "dev": true, + "optional": true + }, "duplexify": { "version": "3.6.1", "from": "duplexify@>=3.6.0 <4.0.0", @@ -657,6 +712,12 @@ } } }, + "fs.realpath": { + "version": "1.0.0", + "from": "fs.realpath@>=1.0.0 <2.0.0", + "resolved": "https://registry.npmjs.org/fs.realpath/-/fs.realpath-1.0.0.tgz", + "dev": true + }, "function-bind": { "version": "1.1.1", "from": "function-bind@>=1.1.1 <2.0.0", @@ -935,6 +996,12 @@ "resolved": "https://registry.npmjs.org/hawk/-/hawk-1.0.0.tgz", "optional": true }, + "he": { + "version": "1.1.1", + "from": "he@1.1.1", + "resolved": "https://registry.npmjs.org/he/-/he-1.1.1.tgz", + "dev": true + }, "hex2dec": { "version": "1.1.1", "from": "hex2dec@>=1.0.1 <2.0.0", @@ -1291,9 +1358,9 @@ "resolved": "https://registry.npmjs.org/methods/-/methods-1.1.2.tgz" }, "metrics-sharelatex": { - "version": "2.0.4", - "from": "git+https://github.com/sharelatex/metrics-sharelatex.git#v2.0.4", - "resolved": "git+https://github.com/sharelatex/metrics-sharelatex.git#49c4bc072f707eb6252adfec540268dfbea1615b", + "version": "2.0.5", + "from": "git+https://github.com/sharelatex/metrics-sharelatex.git#v2.0.5", + "resolved": "git+https://github.com/sharelatex/metrics-sharelatex.git#e5acf25978395e3e0b0333c8b2a047e06435df79", "dependencies": { "coffee-script": { "version": "1.6.0", @@ -1337,6 +1404,56 @@ "from": "mkdirp@~0.5.1", "resolved": "https://registry.npmjs.org/mkdirp/-/mkdirp-0.5.1.tgz" }, + "mocha": { + "version": "4.1.0", + "from": "mocha@>=4.0.1 <5.0.0", + "resolved": "https://registry.npmjs.org/mocha/-/mocha-4.1.0.tgz", + "dev": true, + "dependencies": { + "commander": { + "version": "2.11.0", + "from": "commander@2.11.0", + "resolved": "https://registry.npmjs.org/commander/-/commander-2.11.0.tgz", + "dev": true + }, + "debug": { + "version": "3.1.0", + "from": "debug@3.1.0", + "resolved": "https://registry.npmjs.org/debug/-/debug-3.1.0.tgz", + "dev": true + }, + "diff": { + "version": "3.3.1", + "from": "diff@3.3.1", + "resolved": "https://registry.npmjs.org/diff/-/diff-3.3.1.tgz", + "dev": true + }, + "glob": { + "version": "7.1.2", + "from": "glob@7.1.2", + "resolved": "https://registry.npmjs.org/glob/-/glob-7.1.2.tgz", + "dev": true + }, + "growl": { + "version": "1.10.3", + "from": "growl@1.10.3", + "resolved": "https://registry.npmjs.org/growl/-/growl-1.10.3.tgz", + "dev": true + }, + "has-flag": { + "version": "2.0.0", + "from": "has-flag@>=2.0.0 <3.0.0", + "resolved": "https://registry.npmjs.org/has-flag/-/has-flag-2.0.0.tgz", + "dev": true + }, + "supports-color": { + "version": "4.4.0", + "from": "supports-color@4.4.0", + "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-4.4.0.tgz", + "dev": true + } + } + }, "module-details-from-path": { "version": "1.0.3", "from": "module-details-from-path@>=1.0.3 <2.0.0", @@ -1359,6 +1476,12 @@ "resolved": "https://registry.npmjs.org/nan/-/nan-2.10.0.tgz", "optional": true }, + "native-or-bluebird": { + "version": "1.1.2", + "from": "native-or-bluebird@>=1.1.2 <1.2.0", + "resolved": "https://registry.npmjs.org/native-or-bluebird/-/native-or-bluebird-1.1.2.tgz", + "dev": true + }, "ncp": { "version": "2.0.0", "from": "ncp@~2.0.0", @@ -1673,6 +1796,20 @@ "from": "samsam@1.3.0", "resolved": "https://registry.npmjs.org/samsam/-/samsam-1.3.0.tgz" }, + "sandboxed-module": { + "version": "0.3.0", + "from": "sandboxed-module@>=0.3.0 <0.4.0", + "resolved": "https://registry.npmjs.org/sandboxed-module/-/sandboxed-module-0.3.0.tgz", + "dev": true, + "dependencies": { + "stack-trace": { + "version": "0.0.6", + "from": "stack-trace@0.0.6", + "resolved": "https://registry.npmjs.org/stack-trace/-/stack-trace-0.0.6.tgz", + "dev": true + } + } + }, "semver": { "version": "5.6.0", "from": "semver@>=5.5.0 <6.0.0", @@ -1727,6 +1864,12 @@ "from": "sigmund@>=1.0.0 <1.1.0", "resolved": "https://registry.npmjs.org/sigmund/-/sigmund-1.0.1.tgz" }, + "sinon": { + "version": "1.5.2", + "from": "sinon@>=1.5.2 <1.6.0", + "resolved": "https://registry.npmjs.org/sinon/-/sinon-1.5.2.tgz", + "dev": true + }, "sntp": { "version": "0.2.4", "from": "sntp@>=0.2.0 <0.3.0", @@ -1833,6 +1976,12 @@ "from": "through2@>=2.0.3 <3.0.0", "resolved": "https://registry.npmjs.org/through2/-/through2-2.0.5.tgz" }, + "timekeeper": { + "version": "0.0.4", + "from": "timekeeper@0.0.4", + "resolved": "https://registry.npmjs.org/timekeeper/-/timekeeper-0.0.4.tgz", + "dev": true + }, "tinycolor": { "version": "0.0.1", "from": "tinycolor@>=0.0.0 <1.0.0", @@ -1850,6 +1999,12 @@ "resolved": "https://registry.npmjs.org/tunnel-agent/-/tunnel-agent-0.3.0.tgz", "optional": true }, + "type-detect": { + "version": "0.1.1", + "from": "type-detect@0.1.1", + "resolved": "https://registry.npmjs.org/type-detect/-/type-detect-0.1.1.tgz", + "dev": true + }, "type-is": { "version": "1.6.16", "from": "type-is@>=1.6.16 <1.7.0", @@ -1860,6 +2015,12 @@ "from": "uglify-js@1.2.5", "resolved": "https://registry.npmjs.org/uglify-js/-/uglify-js-1.2.5.tgz" }, + "uid-safe": { + "version": "1.1.0", + "from": "uid-safe@>=1.0.1 <2.0.0", + "resolved": "https://registry.npmjs.org/uid-safe/-/uid-safe-1.1.0.tgz", + "dev": true + }, "underscore": { "version": "1.7.0", "from": "underscore@>=1.7.0 <1.8.0", diff --git a/services/real-time/package.json b/services/real-time/package.json index a2466ea67f..a606f51b24 100644 --- a/services/real-time/package.json +++ b/services/real-time/package.json @@ -29,7 +29,7 @@ "express": "^4.10.1", "express-session": "^1.9.1", "logger-sharelatex": "git+https://github.com/sharelatex/logger-sharelatex.git#v1.5.6", - "metrics-sharelatex": "git+https://github.com/sharelatex/metrics-sharelatex.git#v2.0.4", + "metrics-sharelatex": "git+https://github.com/sharelatex/metrics-sharelatex.git#v2.0.5", "redis-sharelatex": "git+https://github.com/sharelatex/redis-sharelatex.git#v1.0.4", "request": "~2.34.0", "session.socket.io": "^0.1.6", From 78b779a33801c69bdd632b5557171c939c585ac6 Mon Sep 17 00:00:00 2001 From: Henry Oswald Date: Tue, 4 Dec 2018 16:10:03 +0000 Subject: [PATCH 145/491] try different metrics --- services/real-time/npm-shrinkwrap.json | 6 +++--- services/real-time/package.json | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/services/real-time/npm-shrinkwrap.json b/services/real-time/npm-shrinkwrap.json index 8050f90cc7..4bb3cd93f9 100644 --- a/services/real-time/npm-shrinkwrap.json +++ b/services/real-time/npm-shrinkwrap.json @@ -1358,9 +1358,9 @@ "resolved": "https://registry.npmjs.org/methods/-/methods-1.1.2.tgz" }, "metrics-sharelatex": { - "version": "2.0.5", - "from": "git+https://github.com/sharelatex/metrics-sharelatex.git#v2.0.5", - "resolved": "git+https://github.com/sharelatex/metrics-sharelatex.git#e5acf25978395e3e0b0333c8b2a047e06435df79", + "version": "66a57435e602ba5cfff15b50085389fb715907a1", + "from": "git+https://github.com/sharelatex/metrics-sharelatex.git#66a57435e602ba5cfff15b50085389fb715907a1", + "resolved": "git+https://github.com/sharelatex/metrics-sharelatex.git#66a57435e602ba5cfff15b50085389fb715907a1", "dependencies": { "coffee-script": { "version": "1.6.0", diff --git a/services/real-time/package.json b/services/real-time/package.json index a606f51b24..ed75887829 100644 --- a/services/real-time/package.json +++ b/services/real-time/package.json @@ -29,7 +29,7 @@ "express": "^4.10.1", "express-session": "^1.9.1", "logger-sharelatex": "git+https://github.com/sharelatex/logger-sharelatex.git#v1.5.6", - "metrics-sharelatex": "git+https://github.com/sharelatex/metrics-sharelatex.git#v2.0.5", + "metrics-sharelatex": "git+https://github.com/sharelatex/metrics-sharelatex.git#66a57435e602ba5cfff15b50085389fb715907a1", "redis-sharelatex": "git+https://github.com/sharelatex/redis-sharelatex.git#v1.0.4", "request": "~2.34.0", "session.socket.io": "^0.1.6", From cdf605e17117375b67129b7e4cb044250a7c5681 Mon Sep 17 00:00:00 2001 From: Henry Oswald Date: Wed, 5 Dec 2018 14:01:15 +0000 Subject: [PATCH 146/491] inject metrics root and bump lib to 2.0.8 --- services/real-time/app.coffee | 2 ++ services/real-time/npm-shrinkwrap.json | 6 +++--- services/real-time/package.json | 2 +- 3 files changed, 6 insertions(+), 4 deletions(-) diff --git a/services/real-time/app.coffee b/services/real-time/app.coffee index 8d4f8ecac7..94b5594920 100644 --- a/services/real-time/app.coffee +++ b/services/real-time/app.coffee @@ -64,6 +64,8 @@ app.get "/health_check/redis", (req, res, next) -> else res.sendStatus 200 +Metrics.injectMetricsRoute(app) + Router = require "./app/js/Router" Router.configure(app, io, sessionSockets) diff --git a/services/real-time/npm-shrinkwrap.json b/services/real-time/npm-shrinkwrap.json index 4bb3cd93f9..6773b54cf1 100644 --- a/services/real-time/npm-shrinkwrap.json +++ b/services/real-time/npm-shrinkwrap.json @@ -1358,9 +1358,9 @@ "resolved": "https://registry.npmjs.org/methods/-/methods-1.1.2.tgz" }, "metrics-sharelatex": { - "version": "66a57435e602ba5cfff15b50085389fb715907a1", - "from": "git+https://github.com/sharelatex/metrics-sharelatex.git#66a57435e602ba5cfff15b50085389fb715907a1", - "resolved": "git+https://github.com/sharelatex/metrics-sharelatex.git#66a57435e602ba5cfff15b50085389fb715907a1", + "version": "2.0.8", + "from": "git+https://github.com/sharelatex/metrics-sharelatex.git#v2.0.8", + "resolved": "git+https://github.com/sharelatex/metrics-sharelatex.git#a56855e935e4e361166706091cf33f252eb0cd24", "dependencies": { "coffee-script": { "version": "1.6.0", diff --git a/services/real-time/package.json b/services/real-time/package.json index ed75887829..7a842e1b51 100644 --- a/services/real-time/package.json +++ b/services/real-time/package.json @@ -29,7 +29,7 @@ "express": "^4.10.1", "express-session": "^1.9.1", "logger-sharelatex": "git+https://github.com/sharelatex/logger-sharelatex.git#v1.5.6", - "metrics-sharelatex": "git+https://github.com/sharelatex/metrics-sharelatex.git#66a57435e602ba5cfff15b50085389fb715907a1", + "metrics-sharelatex": "git+https://github.com/sharelatex/metrics-sharelatex.git#v2.0.8", "redis-sharelatex": "git+https://github.com/sharelatex/redis-sharelatex.git#v1.0.4", "request": "~2.34.0", "session.socket.io": "^0.1.6", From 5ad7482385e162657728e6bf5ed8f14e7ec208d2 Mon Sep 17 00:00:00 2001 From: Henry Oswald Date: Wed, 5 Dec 2018 14:42:16 +0000 Subject: [PATCH 147/491] update config to take explicit redis configs --- .../real-time/config/settings.defaults.coffee | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/services/real-time/config/settings.defaults.coffee b/services/real-time/config/settings.defaults.coffee index bbe493956f..7839278b9e 100644 --- a/services/real-time/config/settings.defaults.coffee +++ b/services/real-time/config/settings.defaults.coffee @@ -1,24 +1,24 @@ module.exports = redis: realtime: - host: process.env['REDIS_HOST'] or "localhost" - port: process.env['REDIS_PORT'] or "6379" - password: process.env["REDIS_PASSWORD"] or "" + host: process.env['REAL_TIME_REDIS_HOST'] or process.env['REDIS_HOST'] or "localhost" + port: process.env['REAL_TIME_REDIS_PORT'] or process.env['REDIS_PORT'] or "6379" + password: process.env["REAL_TIME_REDIS_PASSWORD"] or process.env["REDIS_PASSWORD"] or "" key_schema: clientsInProject: ({project_id}) -> "clients_in_project:#{project_id}" connectedUser: ({project_id, client_id})-> "connected_user:#{project_id}:#{client_id}" documentupdater: - host: process.env['REDIS_HOST'] or "localhost" - port: process.env['REDIS_PORT'] or "6379" - password: process.env["REDIS_PASSWORD"] or "" + host: process.env['DOC_UPDATER_REDIS_HOST'] or process.env['REDIS_HOST'] or "localhost" + port: process.env['DOC_UPDATER_REDIS_PORT'] or process.env['REDIS_PORT'] or "6379" + password: process.env["DOC_UPDATER_REDIS_PASSWORD"] or process.env["REDIS_PASSWORD"] or "" key_schema: pendingUpdates: ({doc_id}) -> "PendingUpdates:#{doc_id}" websessions: - host: process.env['REDIS_HOST'] or "localhost" - port: process.env['REDIS_PORT'] or "6379" - password: process.env["REDIS_PASSWORD"] or "" + host: process.env['WEB_REDIS_HOST'] or process.env['REDIS_HOST'] or "localhost" + port: process.env['WEB_REDIS_PORT'] or process.env['REDIS_PORT'] or "6379" + password: process.env["WEB_REDIS_PASSWORD"] or process.env["REDIS_PASSWORD"] or "" internal: realTime: From d4e8bc1d4bde42ec80b4151c887c07498058a287 Mon Sep 17 00:00:00 2001 From: Henry Oswald Date: Wed, 5 Dec 2018 15:29:23 +0000 Subject: [PATCH 148/491] remove metrics.set as we don't support it atm --- services/real-time/app/coffee/WebsocketController.coffee | 2 -- 1 file changed, 2 deletions(-) diff --git a/services/real-time/app/coffee/WebsocketController.coffee b/services/real-time/app/coffee/WebsocketController.coffee index b51e86c495..356f757ffb 100644 --- a/services/real-time/app/coffee/WebsocketController.coffee +++ b/services/real-time/app/coffee/WebsocketController.coffee @@ -197,8 +197,6 @@ module.exports = WebsocketController = update.meta.source = client.id update.meta.user_id = user_id metrics.inc "editor.doc-update", 0.3 - metrics.set "editor.active-projects", project_id, 0.3 - metrics.set "editor.active-users", user_id, 0.3 logger.log {user_id, doc_id, project_id, client_id: client.id, version: update.v}, "sending update to doc updater" From 7a7f1aed91583116eb04d0ca0b8c27cd1b9386bc Mon Sep 17 00:00:00 2001 From: Henry Oswald Date: Wed, 5 Dec 2018 15:39:27 +0000 Subject: [PATCH 149/491] remove console.log --- services/real-time/app.coffee | 1 - 1 file changed, 1 deletion(-) diff --git a/services/real-time/app.coffee b/services/real-time/app.coffee index 94b5594920..334d267945 100644 --- a/services/real-time/app.coffee +++ b/services/real-time/app.coffee @@ -52,7 +52,6 @@ app.get "/", (req, res, next) -> res.send "real-time-sharelatex is alive" app.get "/status", (req, res, next) -> - console.log("got status", req.query, io.sockets.clients()?.length) res.send "real-time-sharelatex is alive" rclient = require("redis-sharelatex").createClient(Settings.redis.realtime) From bbd88e75eb6d83fbbb3e3a11395c197c74cdcca3 Mon Sep 17 00:00:00 2001 From: Henry Oswald Date: Wed, 5 Dec 2018 15:41:12 +0000 Subject: [PATCH 150/491] fix broken tests --- .../test/unit/coffee/WebsocketControllerTests.coffee | 6 ------ 1 file changed, 6 deletions(-) diff --git a/services/real-time/test/unit/coffee/WebsocketControllerTests.coffee b/services/real-time/test/unit/coffee/WebsocketControllerTests.coffee index b6a4069c67..9621589d26 100644 --- a/services/real-time/test/unit/coffee/WebsocketControllerTests.coffee +++ b/services/real-time/test/unit/coffee/WebsocketControllerTests.coffee @@ -559,12 +559,6 @@ describe 'WebsocketController', -> it "should call the callback", -> @callback.called.should.equal true - - it "should update the active users metric", -> - @metrics.set.calledWith("editor.active-users", @user_id).should.equal true - - it "should update the active projects metric", -> - @metrics.set.calledWith("editor.active-projects", @project_id).should.equal true it "should increment the doc updates", -> @metrics.inc.calledWith("editor.doc-update").should.equal true From 9e7f84cd0d67f6de226bf4830bbd4d4fb2e64987 Mon Sep 17 00:00:00 2001 From: Christopher Hoskin Date: Thu, 27 Dec 2018 08:23:18 +0000 Subject: [PATCH 151/491] Bump node version to 6.15.1 --- services/real-time/.nvmrc | 2 +- services/real-time/Jenkinsfile | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/services/real-time/.nvmrc b/services/real-time/.nvmrc index 26ec038c18..d36e8d82f3 100644 --- a/services/real-time/.nvmrc +++ b/services/real-time/.nvmrc @@ -1 +1 @@ -6.9.5 \ No newline at end of file +6.15.1 diff --git a/services/real-time/Jenkinsfile b/services/real-time/Jenkinsfile index d908d42063..dd811ae90e 100644 --- a/services/real-time/Jenkinsfile +++ b/services/real-time/Jenkinsfile @@ -15,7 +15,7 @@ pipeline { stage('Install') { agent { docker { - image 'node:6.9.5' + image 'node:6.15.1' args "-v /var/lib/jenkins/.npm:/tmp/.npm -e HOME=/tmp" reuseNode true } @@ -33,7 +33,7 @@ pipeline { stage('Compile and Test') { agent { docker { - image 'node:6.9.5' + image 'node:6.15.1' args "-v /var/lib/jenkins/.npm:/tmp/.npm -e HOME=/tmp" reuseNode true } From cd362f22be4111c506a5efb2b9a58edd1f6aa7a3 Mon Sep 17 00:00:00 2001 From: Christopher Hoskin Date: Thu, 3 Jan 2019 16:17:31 +0000 Subject: [PATCH 152/491] Move to v2 metrics --- services/real-time/app.coffee | 1 + services/real-time/npm-shrinkwrap.json | 331 ++++++++++++++----------- services/real-time/package.json | 2 +- 3 files changed, 184 insertions(+), 150 deletions(-) diff --git a/services/real-time/app.coffee b/services/real-time/app.coffee index 334d267945..931e9e42d3 100644 --- a/services/real-time/app.coffee +++ b/services/real-time/app.coffee @@ -25,6 +25,7 @@ DrainManager = require("./app/js/DrainManager") # Set up socket.io server app = express() +Metrics.injectMetricsRoute(app) server = require('http').createServer(app) io = require('socket.io').listen(server) diff --git a/services/real-time/npm-shrinkwrap.json b/services/real-time/npm-shrinkwrap.json index 6773b54cf1..2374262396 100644 --- a/services/real-time/npm-shrinkwrap.json +++ b/services/real-time/npm-shrinkwrap.json @@ -3,24 +3,41 @@ "version": "0.1.4", "dependencies": { "@google-cloud/common": { - "version": "0.23.0", - "from": "@google-cloud/common@>=0.23.0 <0.24.0", - "resolved": "https://registry.npmjs.org/@google-cloud/common/-/common-0.23.0.tgz" + "version": "0.27.0", + "from": "@google-cloud/common@>=0.27.0 <0.28.0", + "resolved": "https://registry.npmjs.org/@google-cloud/common/-/common-0.27.0.tgz" }, "@google-cloud/debug-agent": { - "version": "3.0.0", + "version": "3.0.1", "from": "@google-cloud/debug-agent@>=3.0.0 <4.0.0", - "resolved": "https://registry.npmjs.org/@google-cloud/debug-agent/-/debug-agent-3.0.0.tgz", + "resolved": "https://registry.npmjs.org/@google-cloud/debug-agent/-/debug-agent-3.0.1.tgz", "dependencies": { "coffeescript": { "version": "2.3.2", "from": "coffeescript@>=2.0.0 <3.0.0", "resolved": "https://registry.npmjs.org/coffeescript/-/coffeescript-2.3.2.tgz" + } + } + }, + "@google-cloud/profiler": { + "version": "0.2.3", + "from": "@google-cloud/profiler@>=0.2.3 <0.3.0", + "resolved": "https://registry.npmjs.org/@google-cloud/profiler/-/profiler-0.2.3.tgz", + "dependencies": { + "@google-cloud/common": { + "version": "0.26.2", + "from": "@google-cloud/common@>=0.26.0 <0.27.0", + "resolved": "https://registry.npmjs.org/@google-cloud/common/-/common-0.26.2.tgz" }, - "lodash": { - "version": "4.17.11", - "from": "lodash@>=4.12.0 <5.0.0", - "resolved": "https://registry.npmjs.org/lodash/-/lodash-4.17.11.tgz" + "nan": { + "version": "2.12.1", + "from": "nan@>=2.11.1 <3.0.0", + "resolved": "https://registry.npmjs.org/nan/-/nan-2.12.1.tgz" + }, + "through2": { + "version": "3.0.0", + "from": "through2@>=3.0.0 <4.0.0", + "resolved": "https://registry.npmjs.org/through2/-/through2-3.0.0.tgz" } } }, @@ -35,36 +52,14 @@ "resolved": "https://registry.npmjs.org/@google-cloud/promisify/-/promisify-0.3.1.tgz" }, "@google-cloud/trace-agent": { - "version": "3.4.0", + "version": "3.5.0", "from": "@google-cloud/trace-agent@>=3.2.0 <4.0.0", - "resolved": "https://registry.npmjs.org/@google-cloud/trace-agent/-/trace-agent-3.4.0.tgz", + "resolved": "https://registry.npmjs.org/@google-cloud/trace-agent/-/trace-agent-3.5.0.tgz", "dependencies": { "@google-cloud/common": { - "version": "0.27.0", - "from": "@google-cloud/common@>=0.27.0 <0.28.0", - "resolved": "https://registry.npmjs.org/@google-cloud/common/-/common-0.27.0.tgz" - }, - "gcp-metadata": { - "version": "0.9.0", - "from": "gcp-metadata@>=0.9.0 <0.10.0", - "resolved": "https://registry.npmjs.org/gcp-metadata/-/gcp-metadata-0.9.0.tgz" - }, - "google-auth-library": { - "version": "2.0.1", - "from": "google-auth-library@>=2.0.0 <3.0.0", - "resolved": "https://registry.npmjs.org/google-auth-library/-/google-auth-library-2.0.1.tgz", - "dependencies": { - "gcp-metadata": { - "version": "0.7.0", - "from": "gcp-metadata@^0.7.0", - "resolved": "https://registry.npmjs.org/gcp-metadata/-/gcp-metadata-0.7.0.tgz" - } - } - }, - "lru-cache": { - "version": "4.1.5", - "from": "lru-cache@^4.1.3", - "resolved": "https://registry.npmjs.org/lru-cache/-/lru-cache-4.1.5.tgz" + "version": "0.28.0", + "from": "@google-cloud/common@>=0.28.0 <0.29.0", + "resolved": "https://registry.npmjs.org/@google-cloud/common/-/common-0.28.0.tgz" }, "uuid": { "version": "3.3.2", @@ -73,6 +68,61 @@ } } }, + "@protobufjs/aspromise": { + "version": "1.1.2", + "from": "@protobufjs/aspromise@>=1.1.2 <2.0.0", + "resolved": "https://registry.npmjs.org/@protobufjs/aspromise/-/aspromise-1.1.2.tgz" + }, + "@protobufjs/base64": { + "version": "1.1.2", + "from": "@protobufjs/base64@>=1.1.2 <2.0.0", + "resolved": "https://registry.npmjs.org/@protobufjs/base64/-/base64-1.1.2.tgz" + }, + "@protobufjs/codegen": { + "version": "2.0.4", + "from": "@protobufjs/codegen@>=2.0.4 <3.0.0", + "resolved": "https://registry.npmjs.org/@protobufjs/codegen/-/codegen-2.0.4.tgz" + }, + "@protobufjs/eventemitter": { + "version": "1.1.0", + "from": "@protobufjs/eventemitter@>=1.1.0 <2.0.0", + "resolved": "https://registry.npmjs.org/@protobufjs/eventemitter/-/eventemitter-1.1.0.tgz" + }, + "@protobufjs/fetch": { + "version": "1.1.0", + "from": "@protobufjs/fetch@>=1.1.0 <2.0.0", + "resolved": "https://registry.npmjs.org/@protobufjs/fetch/-/fetch-1.1.0.tgz" + }, + "@protobufjs/float": { + "version": "1.0.2", + "from": "@protobufjs/float@>=1.0.2 <2.0.0", + "resolved": "https://registry.npmjs.org/@protobufjs/float/-/float-1.0.2.tgz" + }, + "@protobufjs/inquire": { + "version": "1.1.0", + "from": "@protobufjs/inquire@>=1.1.0 <2.0.0", + "resolved": "https://registry.npmjs.org/@protobufjs/inquire/-/inquire-1.1.0.tgz" + }, + "@protobufjs/path": { + "version": "1.1.2", + "from": "@protobufjs/path@>=1.1.2 <2.0.0", + "resolved": "https://registry.npmjs.org/@protobufjs/path/-/path-1.1.2.tgz" + }, + "@protobufjs/pool": { + "version": "1.1.0", + "from": "@protobufjs/pool@>=1.1.0 <2.0.0", + "resolved": "https://registry.npmjs.org/@protobufjs/pool/-/pool-1.1.0.tgz" + }, + "@protobufjs/utf8": { + "version": "1.1.0", + "from": "@protobufjs/utf8@>=1.1.0 <2.0.0", + "resolved": "https://registry.npmjs.org/@protobufjs/utf8/-/utf8-1.1.0.tgz" + }, + "@sindresorhus/is": { + "version": "0.13.0", + "from": "@sindresorhus/is@>=0.13.0 <0.14.0", + "resolved": "https://registry.npmjs.org/@sindresorhus/is/-/is-0.13.0.tgz" + }, "@sinonjs/formatio": { "version": "2.0.0", "from": "@sinonjs/formatio@>=2.0.0 <3.0.0", @@ -83,6 +133,11 @@ "from": "@types/caseless@*", "resolved": "https://registry.npmjs.org/@types/caseless/-/caseless-0.12.1.tgz" }, + "@types/console-log-level": { + "version": "1.4.0", + "from": "@types/console-log-level@>=1.4.0 <2.0.0", + "resolved": "https://registry.npmjs.org/@types/console-log-level/-/console-log-level-1.4.0.tgz" + }, "@types/duplexify": { "version": "3.6.0", "from": "@types/duplexify@>=3.5.0 <4.0.0", @@ -93,16 +148,26 @@ "from": "@types/form-data@*", "resolved": "https://registry.npmjs.org/@types/form-data/-/form-data-2.2.1.tgz" }, + "@types/long": { + "version": "4.0.0", + "from": "@types/long@>=4.0.0 <5.0.0", + "resolved": "https://registry.npmjs.org/@types/long/-/long-4.0.0.tgz" + }, "@types/node": { - "version": "10.12.12", + "version": "10.12.18", "from": "@types/node@*", - "resolved": "https://registry.npmjs.org/@types/node/-/node-10.12.12.tgz" + "resolved": "https://registry.npmjs.org/@types/node/-/node-10.12.18.tgz" }, "@types/request": { "version": "2.48.1", "from": "@types/request@>=2.47.0 <3.0.0", "resolved": "https://registry.npmjs.org/@types/request/-/request-2.48.1.tgz" }, + "@types/semver": { + "version": "5.5.0", + "from": "@types/semver@>=5.5.0 <6.0.0", + "resolved": "https://registry.npmjs.org/@types/semver/-/semver-5.5.0.tgz" + }, "@types/tough-cookie": { "version": "2.3.4", "from": "@types/tough-cookie@*", @@ -230,6 +295,11 @@ "from": "bignumber.js@>=7.0.0 <8.0.0", "resolved": "https://registry.npmjs.org/bignumber.js/-/bignumber.js-7.2.1.tgz" }, + "bindings": { + "version": "1.3.1", + "from": "bindings@>=1.2.1 <2.0.0", + "resolved": "https://registry.npmjs.org/bindings/-/bindings-1.3.1.tgz" + }, "bintrees": { "version": "1.0.1", "from": "bintrees@1.0.1", @@ -426,10 +496,10 @@ "resolved": "https://registry.npmjs.org/deep-eql/-/deep-eql-0.1.3.tgz", "dev": true }, - "define-properties": { - "version": "1.1.3", - "from": "define-properties@>=1.1.2 <2.0.0", - "resolved": "https://registry.npmjs.org/define-properties/-/define-properties-1.1.3.tgz" + "delay": { + "version": "4.1.0", + "from": "delay@>=4.0.1 <5.0.0", + "resolved": "https://registry.npmjs.org/delay/-/delay-4.1.0.tgz" }, "delayed-stream": { "version": "0.0.5", @@ -499,16 +569,6 @@ "from": "ent@>=2.2.0 <3.0.0", "resolved": "https://registry.npmjs.org/ent/-/ent-2.2.0.tgz" }, - "es-abstract": { - "version": "1.12.0", - "from": "es-abstract@>=1.5.1 <2.0.0", - "resolved": "https://registry.npmjs.org/es-abstract/-/es-abstract-1.12.0.tgz" - }, - "es-to-primitive": { - "version": "1.2.0", - "from": "es-to-primitive@>=1.1.1 <2.0.0", - "resolved": "https://registry.npmjs.org/es-to-primitive/-/es-to-primitive-1.2.0.tgz" - }, "es6-promise": { "version": "4.2.5", "from": "es6-promise@>=4.0.3 <5.0.0", @@ -660,9 +720,9 @@ "resolved": "https://registry.npmjs.org/flexbuffer/-/flexbuffer-0.0.6.tgz" }, "follow-redirects": { - "version": "1.5.10", + "version": "1.6.1", "from": "follow-redirects@>=1.3.0 <2.0.0", - "resolved": "https://registry.npmjs.org/follow-redirects/-/follow-redirects-1.5.10.tgz", + "resolved": "https://registry.npmjs.org/follow-redirects/-/follow-redirects-1.6.1.tgz", "dependencies": { "debug": { "version": "3.1.0", @@ -718,15 +778,15 @@ "resolved": "https://registry.npmjs.org/fs.realpath/-/fs.realpath-1.0.0.tgz", "dev": true }, - "function-bind": { - "version": "1.1.1", - "from": "function-bind@>=1.1.1 <2.0.0", - "resolved": "https://registry.npmjs.org/function-bind/-/function-bind-1.1.1.tgz" + "gaxios": { + "version": "1.0.4", + "from": "gaxios@>=1.0.2 <2.0.0", + "resolved": "https://registry.npmjs.org/gaxios/-/gaxios-1.0.4.tgz" }, "gcp-metadata": { - "version": "0.7.0", - "from": "gcp-metadata@>=0.7.0 <0.8.0", - "resolved": "https://registry.npmjs.org/gcp-metadata/-/gcp-metadata-0.7.0.tgz" + "version": "0.9.3", + "from": "gcp-metadata@>=0.9.0 <0.10.0", + "resolved": "https://registry.npmjs.org/gcp-metadata/-/gcp-metadata-0.9.3.tgz" }, "get-func-name": { "version": "2.0.0", @@ -744,33 +804,26 @@ "resolved": "https://registry.npmjs.org/glob/-/glob-6.0.4.tgz" }, "google-auth-library": { - "version": "1.6.1", - "from": "google-auth-library@>=1.6.0 <2.0.0", - "resolved": "https://registry.npmjs.org/google-auth-library/-/google-auth-library-1.6.1.tgz", + "version": "2.0.2", + "from": "google-auth-library@>=2.0.0 <3.0.0", + "resolved": "https://registry.npmjs.org/google-auth-library/-/google-auth-library-2.0.2.tgz", "dependencies": { "gcp-metadata": { - "version": "0.6.3", - "from": "gcp-metadata@>=0.6.3 <0.7.0", - "resolved": "https://registry.npmjs.org/gcp-metadata/-/gcp-metadata-0.6.3.tgz" + "version": "0.7.0", + "from": "gcp-metadata@>=0.7.0 <0.8.0", + "resolved": "https://registry.npmjs.org/gcp-metadata/-/gcp-metadata-0.7.0.tgz" }, "lru-cache": { - "version": "4.1.5", - "from": "lru-cache@>=4.1.3 <5.0.0", - "resolved": "https://registry.npmjs.org/lru-cache/-/lru-cache-4.1.5.tgz" + "version": "5.1.1", + "from": "lru-cache@>=5.0.0 <6.0.0", + "resolved": "https://registry.npmjs.org/lru-cache/-/lru-cache-5.1.1.tgz" } } }, "google-p12-pem": { - "version": "1.0.2", + "version": "1.0.3", "from": "google-p12-pem@>=1.0.0 <2.0.0", - "resolved": "https://registry.npmjs.org/google-p12-pem/-/google-p12-pem-1.0.2.tgz", - "dependencies": { - "pify": { - "version": "3.0.0", - "from": "pify@>=3.0.0 <4.0.0", - "resolved": "https://registry.npmjs.org/pify/-/pify-3.0.0.tgz" - } - } + "resolved": "https://registry.npmjs.org/google-p12-pem/-/google-p12-pem-1.0.3.tgz" }, "graceful-fs": { "version": "1.2.3", @@ -970,11 +1023,6 @@ } } }, - "has": { - "version": "1.0.3", - "from": "has@>=1.0.1 <2.0.0", - "resolved": "https://registry.npmjs.org/has/-/has-1.0.3.tgz" - }, "has-ansi": { "version": "0.1.0", "from": "has-ansi@>=0.1.0 <0.2.0", @@ -985,11 +1033,6 @@ "from": "has-flag@>=3.0.0 <4.0.0", "resolved": "https://registry.npmjs.org/has-flag/-/has-flag-3.0.0.tgz" }, - "has-symbols": { - "version": "1.0.0", - "from": "has-symbols@>=1.0.0 <2.0.0", - "resolved": "https://registry.npmjs.org/has-symbols/-/has-symbols-1.0.0.tgz" - }, "hawk": { "version": "1.0.0", "from": "hawk@>=1.0.0 <1.1.0", @@ -1071,35 +1114,15 @@ "resolved": "https://registry.npmjs.org/ipaddr.js/-/ipaddr.js-1.6.0.tgz" }, "is": { - "version": "3.2.1", - "from": "is@>=3.2.1 <4.0.0", - "resolved": "https://registry.npmjs.org/is/-/is-3.2.1.tgz" + "version": "3.3.0", + "from": "is@>=3.2.0 <4.0.0", + "resolved": "https://registry.npmjs.org/is/-/is-3.3.0.tgz" }, "is-buffer": { "version": "1.1.6", "from": "is-buffer@>=1.1.5 <2.0.0", "resolved": "https://registry.npmjs.org/is-buffer/-/is-buffer-1.1.6.tgz" }, - "is-callable": { - "version": "1.1.4", - "from": "is-callable@>=1.1.3 <2.0.0", - "resolved": "https://registry.npmjs.org/is-callable/-/is-callable-1.1.4.tgz" - }, - "is-date-object": { - "version": "1.0.1", - "from": "is-date-object@>=1.0.1 <2.0.0", - "resolved": "https://registry.npmjs.org/is-date-object/-/is-date-object-1.0.1.tgz" - }, - "is-regex": { - "version": "1.0.4", - "from": "is-regex@>=1.0.4 <2.0.0", - "resolved": "https://registry.npmjs.org/is-regex/-/is-regex-1.0.4.tgz" - }, - "is-symbol": { - "version": "1.0.2", - "from": "is-symbol@>=1.0.2 <2.0.0", - "resolved": "https://registry.npmjs.org/is-symbol/-/is-symbol-1.0.2.tgz" - }, "isarray": { "version": "0.0.1", "from": "isarray@0.0.1", @@ -1212,11 +1235,6 @@ "from": "lodash.isempty@>=4.4.0 <5.0.0", "resolved": "https://registry.npmjs.org/lodash.isempty/-/lodash.isempty-4.4.0.tgz" }, - "lodash.isstring": { - "version": "4.0.1", - "from": "lodash.isstring@>=4.0.1 <5.0.0", - "resolved": "https://registry.npmjs.org/lodash.isstring/-/lodash.isstring-4.0.1.tgz" - }, "lodash.keys": { "version": "4.2.0", "from": "lodash.keys@>=4.2.0 <5.0.0", @@ -1237,6 +1255,11 @@ "from": "lodash.pick@>=4.4.0 <5.0.0", "resolved": "https://registry.npmjs.org/lodash.pick/-/lodash.pick-4.4.0.tgz" }, + "lodash.pickby": { + "version": "4.6.0", + "from": "lodash.pickby@>=4.6.0 <5.0.0", + "resolved": "https://registry.npmjs.org/lodash.pickby/-/lodash.pickby-4.6.0.tgz" + }, "lodash.sample": { "version": "4.2.1", "from": "lodash.sample@>=4.2.1 <5.0.0", @@ -1322,6 +1345,11 @@ "from": "lolex@>=2.2.0 <3.0.0", "resolved": "https://registry.npmjs.org/lolex/-/lolex-2.6.0.tgz" }, + "long": { + "version": "4.0.0", + "from": "long@>=4.0.0 <5.0.0", + "resolved": "https://registry.npmjs.org/long/-/long-4.0.0.tgz" + }, "lru-cache": { "version": "2.7.3", "from": "lru-cache@>=2.0.0 <3.0.0", @@ -1358,9 +1386,9 @@ "resolved": "https://registry.npmjs.org/methods/-/methods-1.1.2.tgz" }, "metrics-sharelatex": { - "version": "2.0.8", - "from": "git+https://github.com/sharelatex/metrics-sharelatex.git#v2.0.8", - "resolved": "git+https://github.com/sharelatex/metrics-sharelatex.git#a56855e935e4e361166706091cf33f252eb0cd24", + "version": "2.0.12", + "from": "git+https://github.com/sharelatex/metrics-sharelatex.git#v2.0.12", + "resolved": "git+https://github.com/sharelatex/metrics-sharelatex.git#3ac1621ef049e2f2d88a83b3a41011333d609662", "dependencies": { "coffee-script": { "version": "1.6.0", @@ -1526,16 +1554,6 @@ "resolved": "https://registry.npmjs.org/oauth-sign/-/oauth-sign-0.3.0.tgz", "optional": true }, - "object-keys": { - "version": "1.0.12", - "from": "object-keys@>=1.0.12 <2.0.0", - "resolved": "https://registry.npmjs.org/object-keys/-/object-keys-1.0.12.tgz" - }, - "object.getownpropertydescriptors": { - "version": "2.0.3", - "from": "object.getownpropertydescriptors@>=2.0.3 <3.0.0", - "resolved": "https://registry.npmjs.org/object.getownpropertydescriptors/-/object.getownpropertydescriptors-2.0.3.tgz" - }, "on-finished": { "version": "2.3.0", "from": "on-finished@>=2.3.0 <2.4.0", @@ -1557,15 +1575,25 @@ "resolved": "https://registry.npmjs.org/options/-/options-0.0.6.tgz" }, "p-limit": { - "version": "2.0.0", + "version": "2.1.0", "from": "p-limit@>=2.0.0 <3.0.0", - "resolved": "https://registry.npmjs.org/p-limit/-/p-limit-2.0.0.tgz" + "resolved": "https://registry.npmjs.org/p-limit/-/p-limit-2.1.0.tgz" }, "p-try": { "version": "2.0.0", "from": "p-try@>=2.0.0 <3.0.0", "resolved": "https://registry.npmjs.org/p-try/-/p-try-2.0.0.tgz" }, + "parse-duration": { + "version": "0.1.1", + "from": "parse-duration@>=0.1.1 <0.2.0", + "resolved": "https://registry.npmjs.org/parse-duration/-/parse-duration-0.1.1.tgz" + }, + "parse-ms": { + "version": "2.0.0", + "from": "parse-ms@>=2.0.0 <3.0.0", + "resolved": "https://registry.npmjs.org/parse-ms/-/parse-ms-2.0.0.tgz" + }, "parseurl": { "version": "1.3.2", "from": "parseurl@>=1.3.2 <1.4.0", @@ -1601,26 +1629,31 @@ "from": "policyfile@0.0.4", "resolved": "https://registry.npmjs.org/policyfile/-/policyfile-0.0.4.tgz" }, + "pretty-ms": { + "version": "4.0.0", + "from": "pretty-ms@>=4.0.0 <5.0.0", + "resolved": "https://registry.npmjs.org/pretty-ms/-/pretty-ms-4.0.0.tgz" + }, "process-nextick-args": { "version": "2.0.0", "from": "process-nextick-args@>=2.0.0 <2.1.0", "resolved": "https://registry.npmjs.org/process-nextick-args/-/process-nextick-args-2.0.0.tgz" }, "prom-client": { - "version": "11.2.0", + "version": "11.2.1", "from": "prom-client@>=11.1.3 <12.0.0", - "resolved": "https://registry.npmjs.org/prom-client/-/prom-client-11.2.0.tgz" + "resolved": "https://registry.npmjs.org/prom-client/-/prom-client-11.2.1.tgz" + }, + "protobufjs": { + "version": "6.8.8", + "from": "protobufjs@>=6.8.6 <6.9.0", + "resolved": "https://registry.npmjs.org/protobufjs/-/protobufjs-6.8.8.tgz" }, "proxy-addr": { "version": "2.0.3", "from": "proxy-addr@>=2.0.3 <2.1.0", "resolved": "https://registry.npmjs.org/proxy-addr/-/proxy-addr-2.0.3.tgz" }, - "pseudomap": { - "version": "1.0.2", - "from": "pseudomap@>=1.0.2 <2.0.0", - "resolved": "https://registry.npmjs.org/pseudomap/-/pseudomap-1.0.2.tgz" - }, "punycode": { "version": "1.4.1", "from": "punycode@>=1.4.1 <2.0.0", @@ -1756,9 +1789,9 @@ "resolved": "https://registry.npmjs.org/require-like/-/require-like-0.1.2.tgz" }, "resolve": { - "version": "1.8.1", + "version": "1.9.0", "from": "resolve@>=1.5.0 <2.0.0", - "resolved": "https://registry.npmjs.org/resolve/-/resolve-1.8.1.tgz" + "resolved": "https://registry.npmjs.org/resolve/-/resolve-1.9.0.tgz" }, "retry-axios": { "version": "0.3.2", @@ -1944,6 +1977,11 @@ "from": "supports-color@>=0.2.0 <0.3.0", "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-0.2.0.tgz" }, + "symbol-observable": { + "version": "1.2.0", + "from": "symbol-observable@>=1.2.0 <2.0.0", + "resolved": "https://registry.npmjs.org/symbol-observable/-/symbol-observable-1.2.0.tgz" + }, "tdigest": { "version": "0.1.1", "from": "tdigest@>=0.1.1 <0.2.0", @@ -2041,11 +2079,6 @@ "from": "util-deprecate@>=1.0.1 <1.1.0", "resolved": "https://registry.npmjs.org/util-deprecate/-/util-deprecate-1.0.2.tgz" }, - "util.promisify": { - "version": "1.0.0", - "from": "util.promisify@>=1.0.0 <2.0.0", - "resolved": "https://registry.npmjs.org/util.promisify/-/util.promisify-1.0.0.tgz" - }, "utils-merge": { "version": "1.0.1", "from": "utils-merge@1.0.1", @@ -2099,9 +2132,9 @@ "resolved": "https://registry.npmjs.org/xtend/-/xtend-4.0.1.tgz" }, "yallist": { - "version": "2.1.2", - "from": "yallist@>=2.1.2 <3.0.0", - "resolved": "https://registry.npmjs.org/yallist/-/yallist-2.1.2.tgz" + "version": "3.0.3", + "from": "yallist@>=3.0.2 <4.0.0", + "resolved": "https://registry.npmjs.org/yallist/-/yallist-3.0.3.tgz" }, "zeparser": { "version": "0.0.5", diff --git a/services/real-time/package.json b/services/real-time/package.json index 7a842e1b51..893fa9e1f4 100644 --- a/services/real-time/package.json +++ b/services/real-time/package.json @@ -29,7 +29,7 @@ "express": "^4.10.1", "express-session": "^1.9.1", "logger-sharelatex": "git+https://github.com/sharelatex/logger-sharelatex.git#v1.5.6", - "metrics-sharelatex": "git+https://github.com/sharelatex/metrics-sharelatex.git#v2.0.8", + "metrics-sharelatex": "git+https://github.com/sharelatex/metrics-sharelatex.git#v2.0.12", "redis-sharelatex": "git+https://github.com/sharelatex/redis-sharelatex.git#v1.0.4", "request": "~2.34.0", "session.socket.io": "^0.1.6", From cda65ad1caa1309245def070ffeab527c53f7c0b Mon Sep 17 00:00:00 2001 From: Christopher Hoskin Date: Thu, 3 Jan 2019 16:21:33 +0000 Subject: [PATCH 153/491] Bump build scripts to 1.1.10 --- services/real-time/Dockerfile | 6 +++--- services/real-time/Makefile | 2 +- services/real-time/buildscript.txt | 4 ++-- services/real-time/docker-compose.ci.yml | 2 +- services/real-time/docker-compose.yml | 2 +- services/real-time/package.json | 2 +- 6 files changed, 9 insertions(+), 9 deletions(-) diff --git a/services/real-time/Dockerfile b/services/real-time/Dockerfile index aabf01ad91..e7243c5291 100644 --- a/services/real-time/Dockerfile +++ b/services/real-time/Dockerfile @@ -1,4 +1,4 @@ -FROM node:6.9.5 as app +FROM node:6.15.1 as app WORKDIR /app @@ -12,11 +12,11 @@ COPY . /app RUN npm run compile:all -FROM node:6.9.5 +FROM node:6.15.1 COPY --from=app /app /app WORKDIR /app USER node -CMD ["node","app.js"] +CMD ["node", "--expose-gc", "app.js"] diff --git a/services/real-time/Makefile b/services/real-time/Makefile index 7f5e935f7c..8620b42f06 100644 --- a/services/real-time/Makefile +++ b/services/real-time/Makefile @@ -1,7 +1,7 @@ # This file was auto-generated, do not edit it directly. # Instead run bin/update_build_scripts from # https://github.com/sharelatex/sharelatex-dev-environment -# Version: 1.1.9 +# Version: 1.1.10 BUILD_NUMBER ?= local BRANCH_NAME ?= $(shell git rev-parse --abbrev-ref HEAD) diff --git a/services/real-time/buildscript.txt b/services/real-time/buildscript.txt index dbdae4f5ba..c343ba8891 100644 --- a/services/real-time/buildscript.txt +++ b/services/real-time/buildscript.txt @@ -1,6 +1,6 @@ ---script-version=1.1.9 +--script-version=1.1.10 real-time ---node-version=6.9.5 +--node-version=6.15.1 --acceptance-creds=None --language=coffeescript --dependencies=mongo,redis diff --git a/services/real-time/docker-compose.ci.yml b/services/real-time/docker-compose.ci.yml index 17c4ddd2bf..5ab90e1825 100644 --- a/services/real-time/docker-compose.ci.yml +++ b/services/real-time/docker-compose.ci.yml @@ -1,7 +1,7 @@ # This file was auto-generated, do not edit it directly. # Instead run bin/update_build_scripts from # https://github.com/sharelatex/sharelatex-dev-environment -# Version: 1.1.9 +# Version: 1.1.10 version: "2" diff --git a/services/real-time/docker-compose.yml b/services/real-time/docker-compose.yml index dcbc14e683..aeceafb3f3 100644 --- a/services/real-time/docker-compose.yml +++ b/services/real-time/docker-compose.yml @@ -1,7 +1,7 @@ # This file was auto-generated, do not edit it directly. # Instead run bin/update_build_scripts from # https://github.com/sharelatex/sharelatex-dev-environment -# Version: 1.1.9 +# Version: 1.1.10 version: "2" diff --git a/services/real-time/package.json b/services/real-time/package.json index 893fa9e1f4..23d0b77be3 100644 --- a/services/real-time/package.json +++ b/services/real-time/package.json @@ -8,7 +8,7 @@ "url": "https://github.com/sharelatex/real-time-sharelatex.git" }, "scripts": { - "compile:app": "([ -e app/coffee ] && coffee $COFFEE_OPTIONS -o app/js -c app/coffee || echo 'No CoffeeScript folder to compile') && ( [ -e app.coffee ] && coffee $COFFEE_OPTIONS -c app.coffee || echo 'No CoffeeScript app to compile')", + "compile:app": "([ -e app/coffee ] && coffee -m $COFFEE_OPTIONS -o app/js -c app/coffee || echo 'No CoffeeScript folder to compile') && ( [ -e app.coffee ] && coffee -m $COFFEE_OPTIONS -c app.coffee || echo 'No CoffeeScript app to compile')", "start": "npm run compile:app && node $NODE_APP_OPTIONS app.js", "test:acceptance:_run": "mocha --recursive --reporter spec --timeout 30000 --exit $@ test/acceptance/js", "test:acceptance": "npm run compile:app && npm run compile:acceptance_tests && npm run test:acceptance:_run -- --grep=$MOCHA_GREP", From b7e8bf3c4c037c87a289fd8aecf72a6034be1f73 Mon Sep 17 00:00:00 2001 From: Christopher Hoskin Date: Thu, 3 Jan 2019 17:58:10 +0000 Subject: [PATCH 154/491] Bump logger to 1.5.7 --- services/real-time/npm-shrinkwrap.json | 100 +++++++++++++++---------- services/real-time/package.json | 2 +- 2 files changed, 61 insertions(+), 41 deletions(-) diff --git a/services/real-time/npm-shrinkwrap.json b/services/real-time/npm-shrinkwrap.json index 2374262396..34ebae2f58 100644 --- a/services/real-time/npm-shrinkwrap.json +++ b/services/real-time/npm-shrinkwrap.json @@ -123,10 +123,27 @@ "from": "@sindresorhus/is@>=0.13.0 <0.14.0", "resolved": "https://registry.npmjs.org/@sindresorhus/is/-/is-0.13.0.tgz" }, + "@sinonjs/commons": { + "version": "1.3.0", + "from": "@sinonjs/commons@>=1.2.0 <2.0.0", + "resolved": "https://registry.npmjs.org/@sinonjs/commons/-/commons-1.3.0.tgz", + "dependencies": { + "type-detect": { + "version": "4.0.8", + "from": "type-detect@4.0.8", + "resolved": "https://registry.npmjs.org/type-detect/-/type-detect-4.0.8.tgz" + } + } + }, "@sinonjs/formatio": { - "version": "2.0.0", - "from": "@sinonjs/formatio@>=2.0.0 <3.0.0", - "resolved": "https://registry.npmjs.org/@sinonjs/formatio/-/formatio-2.0.0.tgz" + "version": "3.1.0", + "from": "@sinonjs/formatio@>=3.1.0 <4.0.0", + "resolved": "https://registry.npmjs.org/@sinonjs/formatio/-/formatio-3.1.0.tgz" + }, + "@sinonjs/samsam": { + "version": "3.0.2", + "from": "@sinonjs/samsam@>=3.0.2 <4.0.0", + "resolved": "https://registry.npmjs.org/@sinonjs/samsam/-/samsam-3.0.2.tgz" }, "@types/caseless": { "version": "0.12.1", @@ -225,6 +242,11 @@ "from": "array-flatten@1.1.1", "resolved": "https://registry.npmjs.org/array-flatten/-/array-flatten-1.1.1.tgz" }, + "array-from": { + "version": "2.1.1", + "from": "array-from@>=2.1.1 <3.0.0", + "resolved": "https://registry.npmjs.org/array-from/-/array-from-2.1.1.tgz" + }, "arrify": { "version": "1.0.1", "from": "arrify@>=1.0.1 <2.0.0", @@ -1166,9 +1188,9 @@ "resolved": "https://registry.npmjs.org/jsonfile/-/jsonfile-1.1.1.tgz" }, "just-extend": { - "version": "1.1.27", - "from": "just-extend@>=1.1.27 <2.0.0", - "resolved": "https://registry.npmjs.org/just-extend/-/just-extend-1.1.27.tgz" + "version": "4.0.2", + "from": "just-extend@>=4.0.2 <5.0.0", + "resolved": "https://registry.npmjs.org/just-extend/-/just-extend-4.0.2.tgz" }, "jwa": { "version": "1.1.6", @@ -1276,9 +1298,9 @@ "resolved": "https://registry.npmjs.org/lodash.values/-/lodash.values-4.3.0.tgz" }, "logger-sharelatex": { - "version": "1.5.6", - "from": "git+https://github.com/sharelatex/logger-sharelatex.git#v1.5.6", - "resolved": "git+https://github.com/sharelatex/logger-sharelatex.git#b2956ec56b582b9f4fc8fdda8dc00c06e77c5537", + "version": "1.5.9", + "from": "git+https://github.com/sharelatex/logger-sharelatex.git#v1.5.9", + "resolved": "git+https://github.com/sharelatex/logger-sharelatex.git#e8e1b95052f62e107336053e4a983f81cdbdf589", "dependencies": { "assertion-error": { "version": "1.1.0", @@ -1291,15 +1313,20 @@ "resolved": "https://registry.npmjs.org/bunyan/-/bunyan-1.5.1.tgz" }, "chai": { - "version": "4.1.2", + "version": "4.2.0", "from": "chai@latest", - "resolved": "https://registry.npmjs.org/chai/-/chai-4.1.2.tgz" + "resolved": "https://registry.npmjs.org/chai/-/chai-4.2.0.tgz" }, "deep-eql": { "version": "3.0.1", "from": "deep-eql@>=3.0.0 <4.0.0", "resolved": "https://registry.npmjs.org/deep-eql/-/deep-eql-3.0.1.tgz" }, + "diff": { + "version": "3.5.0", + "from": "diff@>=3.5.0 <4.0.0", + "resolved": "https://registry.npmjs.org/diff/-/diff-3.5.0.tgz" + }, "dtrace-provider": { "version": "0.6.0", "from": "dtrace-provider@>=0.6.0 <0.7.0", @@ -1312,21 +1339,14 @@ "resolved": "https://registry.npmjs.org/sandboxed-module/-/sandboxed-module-2.0.3.tgz" }, "sinon": { - "version": "5.0.7", + "version": "7.2.2", "from": "sinon@latest", - "resolved": "https://registry.npmjs.org/sinon/-/sinon-5.0.7.tgz", - "dependencies": { - "diff": { - "version": "3.5.0", - "from": "diff@>=3.1.0 <4.0.0", - "resolved": "https://registry.npmjs.org/diff/-/diff-3.5.0.tgz" - }, - "supports-color": { - "version": "5.4.0", - "from": "supports-color@>=5.1.0 <6.0.0", - "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-5.4.0.tgz" - } - } + "resolved": "https://registry.npmjs.org/sinon/-/sinon-7.2.2.tgz" + }, + "supports-color": { + "version": "5.5.0", + "from": "supports-color@>=5.5.0 <6.0.0", + "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-5.5.0.tgz" }, "timekeeper": { "version": "1.0.0", @@ -1341,9 +1361,9 @@ } }, "lolex": { - "version": "2.6.0", - "from": "lolex@>=2.2.0 <3.0.0", - "resolved": "https://registry.npmjs.org/lolex/-/lolex-2.6.0.tgz" + "version": "3.0.0", + "from": "lolex@>=3.0.0 <4.0.0", + "resolved": "https://registry.npmjs.org/lolex/-/lolex-3.0.0.tgz" }, "long": { "version": "4.0.0", @@ -1499,9 +1519,9 @@ "optional": true }, "nan": { - "version": "2.10.0", + "version": "2.12.1", "from": "nan@>=2.0.8 <3.0.0", - "resolved": "https://registry.npmjs.org/nan/-/nan-2.10.0.tgz", + "resolved": "https://registry.npmjs.org/nan/-/nan-2.12.1.tgz", "optional": true }, "native-or-bluebird": { @@ -1522,10 +1542,15 @@ "resolved": "https://registry.npmjs.org/negotiator/-/negotiator-0.6.1.tgz" }, "nise": { - "version": "1.3.3", - "from": "nise@>=1.2.0 <2.0.0", - "resolved": "https://registry.npmjs.org/nise/-/nise-1.3.3.tgz", + "version": "1.4.8", + "from": "nise@>=1.4.7 <2.0.0", + "resolved": "https://registry.npmjs.org/nise/-/nise-1.4.8.tgz", "dependencies": { + "lolex": { + "version": "2.7.5", + "from": "lolex@>=2.3.2 <3.0.0", + "resolved": "https://registry.npmjs.org/lolex/-/lolex-2.7.5.tgz" + }, "path-to-regexp": { "version": "1.7.0", "from": "path-to-regexp@>=1.7.0 <2.0.0", @@ -1814,9 +1839,9 @@ "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.1.1.tgz" }, "safe-json-stringify": { - "version": "1.1.0", + "version": "1.2.0", "from": "safe-json-stringify@>=1.0.0 <2.0.0", - "resolved": "https://registry.npmjs.org/safe-json-stringify/-/safe-json-stringify-1.1.0.tgz", + "resolved": "https://registry.npmjs.org/safe-json-stringify/-/safe-json-stringify-1.2.0.tgz", "optional": true }, "safer-buffer": { @@ -1824,11 +1849,6 @@ "from": "safer-buffer@>=2.1.2 <3.0.0", "resolved": "https://registry.npmjs.org/safer-buffer/-/safer-buffer-2.1.2.tgz" }, - "samsam": { - "version": "1.3.0", - "from": "samsam@1.3.0", - "resolved": "https://registry.npmjs.org/samsam/-/samsam-1.3.0.tgz" - }, "sandboxed-module": { "version": "0.3.0", "from": "sandboxed-module@>=0.3.0 <0.4.0", diff --git a/services/real-time/package.json b/services/real-time/package.json index 23d0b77be3..0e5b5ee55e 100644 --- a/services/real-time/package.json +++ b/services/real-time/package.json @@ -28,7 +28,7 @@ "cookie-parser": "^1.3.3", "express": "^4.10.1", "express-session": "^1.9.1", - "logger-sharelatex": "git+https://github.com/sharelatex/logger-sharelatex.git#v1.5.6", + "logger-sharelatex": "git+https://github.com/sharelatex/logger-sharelatex.git#v1.5.9", "metrics-sharelatex": "git+https://github.com/sharelatex/metrics-sharelatex.git#v2.0.12", "redis-sharelatex": "git+https://github.com/sharelatex/redis-sharelatex.git#v1.0.4", "request": "~2.34.0", From c6fa764a6a80f504d2dc7940149f097ca46a6b98 Mon Sep 17 00:00:00 2001 From: Christopher Hoskin Date: Thu, 3 Jan 2019 18:04:50 +0000 Subject: [PATCH 155/491] Add app.js.map to .gitignore --- services/real-time/.gitignore | 1 + 1 file changed, 1 insertion(+) diff --git a/services/real-time/.gitignore b/services/real-time/.gitignore index 7c45322029..22adf4a227 100644 --- a/services/real-time/.gitignore +++ b/services/real-time/.gitignore @@ -4,3 +4,4 @@ app.js app/js test/unit/js test/acceptance/js +app.js.map From 4a3711aba89c91812d0d0a221121e95b753572a9 Mon Sep 17 00:00:00 2001 From: Christopher Hoskin Date: Tue, 8 Jan 2019 14:26:05 +0000 Subject: [PATCH 156/491] Bump settings to v1.1.0 --- services/real-time/npm-shrinkwrap.json | 6 +++--- services/real-time/package.json | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/services/real-time/npm-shrinkwrap.json b/services/real-time/npm-shrinkwrap.json index 34ebae2f58..6b39192931 100644 --- a/services/real-time/npm-shrinkwrap.json +++ b/services/real-time/npm-shrinkwrap.json @@ -1896,9 +1896,9 @@ "resolved": "https://registry.npmjs.org/setprototypeof/-/setprototypeof-1.1.0.tgz" }, "settings-sharelatex": { - "version": "1.0.0", - "from": "git+https://github.com/sharelatex/settings-sharelatex.git#v1.0.0", - "resolved": "git+https://github.com/sharelatex/settings-sharelatex.git#cbc5e41c1dbe6789721a14b3fdae05bf22546559", + "version": "1.1.0", + "from": "git+https://github.com/sharelatex/settings-sharelatex.git#v1.1.0", + "resolved": "git+https://github.com/sharelatex/settings-sharelatex.git#93f63d029b52fef8825c3a401b2b6a7ba29b4750", "dependencies": { "coffee-script": { "version": "1.6.0", diff --git a/services/real-time/package.json b/services/real-time/package.json index 0e5b5ee55e..5277ec536d 100644 --- a/services/real-time/package.json +++ b/services/real-time/package.json @@ -33,7 +33,7 @@ "redis-sharelatex": "git+https://github.com/sharelatex/redis-sharelatex.git#v1.0.4", "request": "~2.34.0", "session.socket.io": "^0.1.6", - "settings-sharelatex": "git+https://github.com/sharelatex/settings-sharelatex.git#v1.0.0", + "settings-sharelatex": "git+https://github.com/sharelatex/settings-sharelatex.git#v1.1.0", "socket.io": "0.9.16", "socket.io-client": "^0.9.16" }, From 3288d6d3211faa77bc666878926f19b585ef20d2 Mon Sep 17 00:00:00 2001 From: Christopher Hoskin Date: Tue, 8 Jan 2019 14:28:10 +0000 Subject: [PATCH 157/491] Add **/*.map to .gitignore --- services/real-time/.gitignore | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/services/real-time/.gitignore b/services/real-time/.gitignore index 22adf4a227..ff0c7e15d2 100644 --- a/services/real-time/.gitignore +++ b/services/real-time/.gitignore @@ -4,4 +4,4 @@ app.js app/js test/unit/js test/acceptance/js -app.js.map +**/*.map From e3618acf208783479c0a482f91f51776a8c88e5a Mon Sep 17 00:00:00 2001 From: Henry Oswald Date: Tue, 22 Jan 2019 15:50:34 +0000 Subject: [PATCH 158/491] mvp2 for redis-cluster --- services/real-time/app.coffee | 2 +- .../real-time/config/settings.defaults.coffee | 28 +++++++++++++++---- services/real-time/npm-shrinkwrap.json | 6 ++-- services/real-time/package.json | 2 +- 4 files changed, 27 insertions(+), 11 deletions(-) diff --git a/services/real-time/app.coffee b/services/real-time/app.coffee index 931e9e42d3..b5eeceedde 100644 --- a/services/real-time/app.coffee +++ b/services/real-time/app.coffee @@ -19,7 +19,7 @@ SessionSockets = require('session.socket.io') CookieParser = require("cookie-parser") - +sessionRedisClient.set "hello-a", "hello-there" DrainManager = require("./app/js/DrainManager") diff --git a/services/real-time/config/settings.defaults.coffee b/services/real-time/config/settings.defaults.coffee index 7839278b9e..2e3a8f46ff 100644 --- a/services/real-time/config/settings.defaults.coffee +++ b/services/real-time/config/settings.defaults.coffee @@ -1,4 +1,4 @@ -module.exports = +settings = redis: realtime: host: process.env['REAL_TIME_REDIS_HOST'] or process.env['REDIS_HOST'] or "localhost" @@ -15,10 +15,9 @@ module.exports = key_schema: pendingUpdates: ({doc_id}) -> "PendingUpdates:#{doc_id}" - websessions: - host: process.env['WEB_REDIS_HOST'] or process.env['REDIS_HOST'] or "localhost" - port: process.env['WEB_REDIS_PORT'] or process.env['REDIS_PORT'] or "6379" - password: process.env["WEB_REDIS_PASSWORD"] or process.env["REDIS_PASSWORD"] or "" + websessions: {} + + internal: realTime: @@ -42,4 +41,21 @@ module.exports = max_doc_length: 2 * 1024 * 1024 # 2mb - forceDrainMsDelay: process.env['FORCE_DRAIN_MS_DELAY'] or false \ No newline at end of file + forceDrainMsDelay: process.env['FORCE_DRAIN_MS_DELAY'] or false + +if process.env['REDIS_CLUSTER_ENABLED'] == "true" + settings.redis.websessions.cluster = [ + { host: process.env["SL_LIN_STAG_REDIS_3_SERVICE_HOST"], port: "6379" } + ] + settings.redis.websessions.natMap = { + '192.168.201.24:6379': { host: process.env["SL_LIN_STAG_REDIS_0_SERVICE_HOST"], port: "6379" } + '192.168.195.231:6379': { host: process.env["SL_LIN_STAG_REDIS_1_SERVICE_HOST"], port: "6379" } + '192.168.223.53:6379': { host: process.env["SL_LIN_STAG_REDIS_2_SERVICE_HOST"], port: "6379" } + '192.168.221.84:6379': { host: process.env["SL_LIN_STAG_REDIS_3_SERVICE_HOST"], port: "6379" } + '192.168.219.81:6379': { host: process.env["SL_LIN_STAG_REDIS_4_SERVICE_HOST"], port: "6379" } + '192.168.180.104:6379': { host: process.env["SL_LIN_STAG_REDIS_5_SERVICE_HOST"], port: "6379" } + '192.168.220.59:6379': { host: process.env["SL_LIN_STAG_REDIS_6_SERVICE_HOST"], port: "6379" } + '192.168.129.122:6379': { host: process.env["SL_LIN_STAG_REDIS_7_SERVICE_HOST"], port: "6379" } + } + +module.exports = settings \ No newline at end of file diff --git a/services/real-time/npm-shrinkwrap.json b/services/real-time/npm-shrinkwrap.json index 6b39192931..2ea4265008 100644 --- a/services/real-time/npm-shrinkwrap.json +++ b/services/real-time/npm-shrinkwrap.json @@ -1755,9 +1755,9 @@ } }, "redis-sharelatex": { - "version": "1.0.4", - "from": "git+https://github.com/sharelatex/redis-sharelatex.git#v1.0.4", - "resolved": "git+https://github.com/sharelatex/redis-sharelatex.git#ca4e906559c1405d132e8edd7db763d64a57be62", + "version": "511b59f3414d6e3db23a4b61239fd2cf3e07cdb0", + "from": "git+https://github.com/sharelatex/redis-sharelatex.git#511b59f3414d6e3db23a4b61239fd2cf3e07cdb0", + "resolved": "git+https://github.com/sharelatex/redis-sharelatex.git#511b59f3414d6e3db23a4b61239fd2cf3e07cdb0", "dependencies": { "async": { "version": "2.6.1", diff --git a/services/real-time/package.json b/services/real-time/package.json index 5277ec536d..d40719df76 100644 --- a/services/real-time/package.json +++ b/services/real-time/package.json @@ -30,7 +30,7 @@ "express-session": "^1.9.1", "logger-sharelatex": "git+https://github.com/sharelatex/logger-sharelatex.git#v1.5.9", "metrics-sharelatex": "git+https://github.com/sharelatex/metrics-sharelatex.git#v2.0.12", - "redis-sharelatex": "git+https://github.com/sharelatex/redis-sharelatex.git#v1.0.4", + "redis-sharelatex": "git+https://github.com/sharelatex/redis-sharelatex.git#511b59f3414d6e3db23a4b61239fd2cf3e07cdb0", "request": "~2.34.0", "session.socket.io": "^0.1.6", "settings-sharelatex": "git+https://github.com/sharelatex/settings-sharelatex.git#v1.1.0", From 9517aac86dd5d5c0a3a14efae267aa564194f479 Mon Sep 17 00:00:00 2001 From: Henry Oswald Date: Tue, 22 Jan 2019 16:02:13 +0000 Subject: [PATCH 159/491] bump redis version --- services/real-time/npm-shrinkwrap.json | 6 +++--- services/real-time/package.json | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/services/real-time/npm-shrinkwrap.json b/services/real-time/npm-shrinkwrap.json index 2ea4265008..ec3f130eae 100644 --- a/services/real-time/npm-shrinkwrap.json +++ b/services/real-time/npm-shrinkwrap.json @@ -1755,9 +1755,9 @@ } }, "redis-sharelatex": { - "version": "511b59f3414d6e3db23a4b61239fd2cf3e07cdb0", - "from": "git+https://github.com/sharelatex/redis-sharelatex.git#511b59f3414d6e3db23a4b61239fd2cf3e07cdb0", - "resolved": "git+https://github.com/sharelatex/redis-sharelatex.git#511b59f3414d6e3db23a4b61239fd2cf3e07cdb0", + "version": "f5956885d59938f1972c75865bedffa8914fdf63", + "from": "git+https://github.com/sharelatex/redis-sharelatex.git#f5956885d59938f1972c75865bedffa8914fdf63", + "resolved": "git+https://github.com/sharelatex/redis-sharelatex.git#f5956885d59938f1972c75865bedffa8914fdf63", "dependencies": { "async": { "version": "2.6.1", diff --git a/services/real-time/package.json b/services/real-time/package.json index d40719df76..53af47e3c8 100644 --- a/services/real-time/package.json +++ b/services/real-time/package.json @@ -30,7 +30,7 @@ "express-session": "^1.9.1", "logger-sharelatex": "git+https://github.com/sharelatex/logger-sharelatex.git#v1.5.9", "metrics-sharelatex": "git+https://github.com/sharelatex/metrics-sharelatex.git#v2.0.12", - "redis-sharelatex": "git+https://github.com/sharelatex/redis-sharelatex.git#511b59f3414d6e3db23a4b61239fd2cf3e07cdb0", + "redis-sharelatex": "git+https://github.com/sharelatex/redis-sharelatex.git#f5956885d59938f1972c75865bedffa8914fdf63", "request": "~2.34.0", "session.socket.io": "^0.1.6", "settings-sharelatex": "git+https://github.com/sharelatex/settings-sharelatex.git#v1.1.0", From 150483eeceff23ae48b640fc249d97d174e23631 Mon Sep 17 00:00:00 2001 From: Henry Oswald Date: Tue, 22 Jan 2019 16:44:39 +0000 Subject: [PATCH 160/491] put redis keys back in for web --- services/real-time/config/settings.defaults.coffee | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/services/real-time/config/settings.defaults.coffee b/services/real-time/config/settings.defaults.coffee index 2e3a8f46ff..9e7f567c78 100644 --- a/services/real-time/config/settings.defaults.coffee +++ b/services/real-time/config/settings.defaults.coffee @@ -15,7 +15,10 @@ settings = key_schema: pendingUpdates: ({doc_id}) -> "PendingUpdates:#{doc_id}" - websessions: {} + websessions: + host: process.env['WEB_REDIS_HOST'] or process.env['REDIS_HOST'] or "localhost" + port: process.env['WEB_REDIS_PORT'] or process.env['REDIS_PORT'] or "6379" + password: process.env["WEB_REDIS_PASSWORD"] or process.env["REDIS_PASSWORD"] or "" From 2cd3e6f18f1879dc53e6ed1cc1cdd0641b3a5c54 Mon Sep 17 00:00:00 2001 From: Henry Oswald Date: Tue, 22 Jan 2019 16:49:22 +0000 Subject: [PATCH 161/491] added logging in settings --- services/real-time/config/settings.defaults.coffee | 2 ++ 1 file changed, 2 insertions(+) diff --git a/services/real-time/config/settings.defaults.coffee b/services/real-time/config/settings.defaults.coffee index 9e7f567c78..bb4d235e74 100644 --- a/services/real-time/config/settings.defaults.coffee +++ b/services/real-time/config/settings.defaults.coffee @@ -46,7 +46,9 @@ settings = forceDrainMsDelay: process.env['FORCE_DRAIN_MS_DELAY'] or false +console.log "process.env['REDIS_CLUSTER_ENABLED']", process.env['REDIS_CLUSTER_ENABLED'] if process.env['REDIS_CLUSTER_ENABLED'] == "true" + settings.redis.websessions.cluster = [ { host: process.env["SL_LIN_STAG_REDIS_3_SERVICE_HOST"], port: "6379" } ] From 52014c851bdd18bf0886c7f9ebc435aec0390d09 Mon Sep 17 00:00:00 2001 From: Henry Oswald Date: Tue, 22 Jan 2019 17:00:48 +0000 Subject: [PATCH 162/491] bump redis --- services/real-time/npm-shrinkwrap.json | 11 +++-------- services/real-time/package.json | 2 +- 2 files changed, 4 insertions(+), 9 deletions(-) diff --git a/services/real-time/npm-shrinkwrap.json b/services/real-time/npm-shrinkwrap.json index ec3f130eae..356b9e51f4 100644 --- a/services/real-time/npm-shrinkwrap.json +++ b/services/real-time/npm-shrinkwrap.json @@ -1125,11 +1125,6 @@ "from": "inherits@2.0.3", "resolved": "https://registry.npmjs.org/inherits/-/inherits-2.0.3.tgz" }, - "ioredis": { - "version": "3.2.2", - "from": "ioredis@>=3.2.1 <4.0.0", - "resolved": "https://registry.npmjs.org/ioredis/-/ioredis-3.2.2.tgz" - }, "ipaddr.js": { "version": "1.6.0", "from": "ipaddr.js@1.6.0", @@ -1755,9 +1750,9 @@ } }, "redis-sharelatex": { - "version": "f5956885d59938f1972c75865bedffa8914fdf63", - "from": "git+https://github.com/sharelatex/redis-sharelatex.git#f5956885d59938f1972c75865bedffa8914fdf63", - "resolved": "git+https://github.com/sharelatex/redis-sharelatex.git#f5956885d59938f1972c75865bedffa8914fdf63", + "version": "4156cfd9ba9444d87a599f6d7035b73b505070f2", + "from": "git+https://github.com/sharelatex/redis-sharelatex.git#4156cfd9ba9444d87a599f6d7035b73b505070f2", + "resolved": "git+https://github.com/sharelatex/redis-sharelatex.git#4156cfd9ba9444d87a599f6d7035b73b505070f2", "dependencies": { "async": { "version": "2.6.1", diff --git a/services/real-time/package.json b/services/real-time/package.json index 53af47e3c8..e5aa8ee54d 100644 --- a/services/real-time/package.json +++ b/services/real-time/package.json @@ -30,7 +30,7 @@ "express-session": "^1.9.1", "logger-sharelatex": "git+https://github.com/sharelatex/logger-sharelatex.git#v1.5.9", "metrics-sharelatex": "git+https://github.com/sharelatex/metrics-sharelatex.git#v2.0.12", - "redis-sharelatex": "git+https://github.com/sharelatex/redis-sharelatex.git#f5956885d59938f1972c75865bedffa8914fdf63", + "redis-sharelatex": "git+https://github.com/sharelatex/redis-sharelatex.git#4156cfd9ba9444d87a599f6d7035b73b505070f2", "request": "~2.34.0", "session.socket.io": "^0.1.6", "settings-sharelatex": "git+https://github.com/sharelatex/settings-sharelatex.git#v1.1.0", From 397b65abf70cf8d62bd0d8735cd6ba075949a7a1 Mon Sep 17 00:00:00 2001 From: Henry Oswald Date: Tue, 22 Jan 2019 17:06:43 +0000 Subject: [PATCH 163/491] put settings on all redis's --- services/real-time/config/settings.defaults.coffee | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/services/real-time/config/settings.defaults.coffee b/services/real-time/config/settings.defaults.coffee index bb4d235e74..485403ca4d 100644 --- a/services/real-time/config/settings.defaults.coffee +++ b/services/real-time/config/settings.defaults.coffee @@ -49,10 +49,10 @@ settings = console.log "process.env['REDIS_CLUSTER_ENABLED']", process.env['REDIS_CLUSTER_ENABLED'] if process.env['REDIS_CLUSTER_ENABLED'] == "true" - settings.redis.websessions.cluster = [ + settings.redis.realtime.cluster = settings.redis.documentupdater.cluster = settings.redis.websessions.cluster = [ { host: process.env["SL_LIN_STAG_REDIS_3_SERVICE_HOST"], port: "6379" } ] - settings.redis.websessions.natMap = { + settings.redis.realtime.natMap = settings.redis.documentupdater.natMap = settings.redis.websessions.natMap = { '192.168.201.24:6379': { host: process.env["SL_LIN_STAG_REDIS_0_SERVICE_HOST"], port: "6379" } '192.168.195.231:6379': { host: process.env["SL_LIN_STAG_REDIS_1_SERVICE_HOST"], port: "6379" } '192.168.223.53:6379': { host: process.env["SL_LIN_STAG_REDIS_2_SERVICE_HOST"], port: "6379" } @@ -62,5 +62,6 @@ if process.env['REDIS_CLUSTER_ENABLED'] == "true" '192.168.220.59:6379': { host: process.env["SL_LIN_STAG_REDIS_6_SERVICE_HOST"], port: "6379" } '192.168.129.122:6379': { host: process.env["SL_LIN_STAG_REDIS_7_SERVICE_HOST"], port: "6379" } } + settings.redis.documentupdater.cluster module.exports = settings \ No newline at end of file From 73b2c1ec05b4db1edc56c0b51a6ae3063141d964 Mon Sep 17 00:00:00 2001 From: Henry Oswald Date: Wed, 23 Jan 2019 10:11:26 +0000 Subject: [PATCH 164/491] auto wrap redis from env vars --- .../real-time/config/settings.defaults.coffee | 36 +++++++++---------- 1 file changed, 17 insertions(+), 19 deletions(-) diff --git a/services/real-time/config/settings.defaults.coffee b/services/real-time/config/settings.defaults.coffee index 485403ca4d..2e525adb00 100644 --- a/services/real-time/config/settings.defaults.coffee +++ b/services/real-time/config/settings.defaults.coffee @@ -7,6 +7,8 @@ settings = key_schema: clientsInProject: ({project_id}) -> "clients_in_project:#{project_id}" connectedUser: ({project_id, client_id})-> "connected_user:#{project_id}:#{client_id}" + cluster: process.env['REDIS_CLUSTER_HOSTS'] + natMap: process.env['REDIS_CLUSTER_NATMAP'] documentupdater: host: process.env['DOC_UPDATER_REDIS_HOST'] or process.env['REDIS_HOST'] or "localhost" @@ -14,13 +16,15 @@ settings = password: process.env["DOC_UPDATER_REDIS_PASSWORD"] or process.env["REDIS_PASSWORD"] or "" key_schema: pendingUpdates: ({doc_id}) -> "PendingUpdates:#{doc_id}" + cluster: process.env['REDIS_CLUSTER_HOSTS'] + natMap: process.env['REDIS_CLUSTER_NATMAP'] websessions: host: process.env['WEB_REDIS_HOST'] or process.env['REDIS_HOST'] or "localhost" port: process.env['WEB_REDIS_PORT'] or process.env['REDIS_PORT'] or "6379" password: process.env["WEB_REDIS_PASSWORD"] or process.env["REDIS_PASSWORD"] or "" - - + cluster: process.env['REDIS_CLUSTER_HOSTS'] + natMap: process.env['REDIS_CLUSTER_NATMAP'] internal: realTime: @@ -46,22 +50,16 @@ settings = forceDrainMsDelay: process.env['FORCE_DRAIN_MS_DELAY'] or false -console.log "process.env['REDIS_CLUSTER_ENABLED']", process.env['REDIS_CLUSTER_ENABLED'] -if process.env['REDIS_CLUSTER_ENABLED'] == "true" +console.log process.env['REDIS_CLUSTER_HOSTS'], process.env['REDIS_CLUSTER_NATMAP'] +if process.env['REDIS_CLUSTER_HOSTS']? or process.env['REDIS_CLUSTER_NATMAP']? - settings.redis.realtime.cluster = settings.redis.documentupdater.cluster = settings.redis.websessions.cluster = [ - { host: process.env["SL_LIN_STAG_REDIS_3_SERVICE_HOST"], port: "6379" } - ] - settings.redis.realtime.natMap = settings.redis.documentupdater.natMap = settings.redis.websessions.natMap = { - '192.168.201.24:6379': { host: process.env["SL_LIN_STAG_REDIS_0_SERVICE_HOST"], port: "6379" } - '192.168.195.231:6379': { host: process.env["SL_LIN_STAG_REDIS_1_SERVICE_HOST"], port: "6379" } - '192.168.223.53:6379': { host: process.env["SL_LIN_STAG_REDIS_2_SERVICE_HOST"], port: "6379" } - '192.168.221.84:6379': { host: process.env["SL_LIN_STAG_REDIS_3_SERVICE_HOST"], port: "6379" } - '192.168.219.81:6379': { host: process.env["SL_LIN_STAG_REDIS_4_SERVICE_HOST"], port: "6379" } - '192.168.180.104:6379': { host: process.env["SL_LIN_STAG_REDIS_5_SERVICE_HOST"], port: "6379" } - '192.168.220.59:6379': { host: process.env["SL_LIN_STAG_REDIS_6_SERVICE_HOST"], port: "6379" } - '192.168.129.122:6379': { host: process.env["SL_LIN_STAG_REDIS_7_SERVICE_HOST"], port: "6379" } - } - settings.redis.documentupdater.cluster - + for redisKey in Object.keys(settings.redis) + + if process.env['REDIS_CLUSTER_HOSTS']? + settings.redis[redisKey].cluster = JSON.parse(process.env['REDIS_CLUSTER_HOSTS']) + + + if process.env['REDIS_CLUSTER_NATMAP']? + settings.redis[redisKey].natMap = JSON.parse(process.env['REDIS_CLUSTER_NATMAP']) +console.log settings module.exports = settings \ No newline at end of file From e1b742c2150c08de6003b8461247761998235c49 Mon Sep 17 00:00:00 2001 From: Henry Oswald Date: Wed, 23 Jan 2019 10:15:00 +0000 Subject: [PATCH 165/491] add logging --- services/real-time/config/settings.defaults.coffee | 1 + 1 file changed, 1 insertion(+) diff --git a/services/real-time/config/settings.defaults.coffee b/services/real-time/config/settings.defaults.coffee index 2e525adb00..9eb738676f 100644 --- a/services/real-time/config/settings.defaults.coffee +++ b/services/real-time/config/settings.defaults.coffee @@ -51,6 +51,7 @@ settings = forceDrainMsDelay: process.env['FORCE_DRAIN_MS_DELAY'] or false console.log process.env['REDIS_CLUSTER_HOSTS'], process.env['REDIS_CLUSTER_NATMAP'] +console.log typeof(process.env['REDIS_CLUSTER_HOSTS']) if process.env['REDIS_CLUSTER_HOSTS']? or process.env['REDIS_CLUSTER_NATMAP']? for redisKey in Object.keys(settings.redis) From b193045a2a69de16df6d9bf27783f405885dfafc Mon Sep 17 00:00:00 2001 From: Henry Oswald Date: Fri, 25 Jan 2019 10:04:26 +0000 Subject: [PATCH 166/491] log out real time settings --- services/real-time/app.coffee | 2 ++ 1 file changed, 2 insertions(+) diff --git a/services/real-time/app.coffee b/services/real-time/app.coffee index b5eeceedde..ca7cacfda3 100644 --- a/services/real-time/app.coffee +++ b/services/real-time/app.coffee @@ -2,6 +2,8 @@ Metrics = require("metrics-sharelatex") Settings = require "settings-sharelatex" Metrics.initialize(Settings.appName or "real-time") +console.log Settings.redis + logger = require "logger-sharelatex" logger.initialize("real-time-sharelatex") Metrics.event_loop.monitor(logger) From 2b1f67b6fae173d3c0e7f934c9ed2d990bebe12e Mon Sep 17 00:00:00 2001 From: Henry Oswald Date: Fri, 25 Jan 2019 10:30:31 +0000 Subject: [PATCH 167/491] remove natmap from defaults config --- .../real-time/config/settings.defaults.coffee | 15 ++------------- 1 file changed, 2 insertions(+), 13 deletions(-) diff --git a/services/real-time/config/settings.defaults.coffee b/services/real-time/config/settings.defaults.coffee index 9eb738676f..46297b9dfc 100644 --- a/services/real-time/config/settings.defaults.coffee +++ b/services/real-time/config/settings.defaults.coffee @@ -50,17 +50,6 @@ settings = forceDrainMsDelay: process.env['FORCE_DRAIN_MS_DELAY'] or false -console.log process.env['REDIS_CLUSTER_HOSTS'], process.env['REDIS_CLUSTER_NATMAP'] -console.log typeof(process.env['REDIS_CLUSTER_HOSTS']) -if process.env['REDIS_CLUSTER_HOSTS']? or process.env['REDIS_CLUSTER_NATMAP']? - - for redisKey in Object.keys(settings.redis) - - if process.env['REDIS_CLUSTER_HOSTS']? - settings.redis[redisKey].cluster = JSON.parse(process.env['REDIS_CLUSTER_HOSTS']) - - - if process.env['REDIS_CLUSTER_NATMAP']? - settings.redis[redisKey].natMap = JSON.parse(process.env['REDIS_CLUSTER_NATMAP']) -console.log settings + +# console.log settings.redis module.exports = settings \ No newline at end of file From 01d3d9d13ed6c6605ae9b924594d81c68393e017 Mon Sep 17 00:00:00 2001 From: Henry Oswald Date: Fri, 25 Jan 2019 10:37:00 +0000 Subject: [PATCH 168/491] remove natmap --- services/real-time/config/settings.defaults.coffee | 3 --- 1 file changed, 3 deletions(-) diff --git a/services/real-time/config/settings.defaults.coffee b/services/real-time/config/settings.defaults.coffee index 46297b9dfc..337c116e14 100644 --- a/services/real-time/config/settings.defaults.coffee +++ b/services/real-time/config/settings.defaults.coffee @@ -8,7 +8,6 @@ settings = clientsInProject: ({project_id}) -> "clients_in_project:#{project_id}" connectedUser: ({project_id, client_id})-> "connected_user:#{project_id}:#{client_id}" cluster: process.env['REDIS_CLUSTER_HOSTS'] - natMap: process.env['REDIS_CLUSTER_NATMAP'] documentupdater: host: process.env['DOC_UPDATER_REDIS_HOST'] or process.env['REDIS_HOST'] or "localhost" @@ -17,14 +16,12 @@ settings = key_schema: pendingUpdates: ({doc_id}) -> "PendingUpdates:#{doc_id}" cluster: process.env['REDIS_CLUSTER_HOSTS'] - natMap: process.env['REDIS_CLUSTER_NATMAP'] websessions: host: process.env['WEB_REDIS_HOST'] or process.env['REDIS_HOST'] or "localhost" port: process.env['WEB_REDIS_PORT'] or process.env['REDIS_PORT'] or "6379" password: process.env["WEB_REDIS_PASSWORD"] or process.env["REDIS_PASSWORD"] or "" cluster: process.env['REDIS_CLUSTER_HOSTS'] - natMap: process.env['REDIS_CLUSTER_NATMAP'] internal: realTime: From 338bee061cee5992362b19457810027ec52ea98c Mon Sep 17 00:00:00 2001 From: Henry Oswald Date: Fri, 25 Jan 2019 10:50:34 +0000 Subject: [PATCH 169/491] remove the cluster key as well --- services/real-time/config/settings.defaults.coffee | 3 --- 1 file changed, 3 deletions(-) diff --git a/services/real-time/config/settings.defaults.coffee b/services/real-time/config/settings.defaults.coffee index 337c116e14..4f48c3bcb3 100644 --- a/services/real-time/config/settings.defaults.coffee +++ b/services/real-time/config/settings.defaults.coffee @@ -7,7 +7,6 @@ settings = key_schema: clientsInProject: ({project_id}) -> "clients_in_project:#{project_id}" connectedUser: ({project_id, client_id})-> "connected_user:#{project_id}:#{client_id}" - cluster: process.env['REDIS_CLUSTER_HOSTS'] documentupdater: host: process.env['DOC_UPDATER_REDIS_HOST'] or process.env['REDIS_HOST'] or "localhost" @@ -15,13 +14,11 @@ settings = password: process.env["DOC_UPDATER_REDIS_PASSWORD"] or process.env["REDIS_PASSWORD"] or "" key_schema: pendingUpdates: ({doc_id}) -> "PendingUpdates:#{doc_id}" - cluster: process.env['REDIS_CLUSTER_HOSTS'] websessions: host: process.env['WEB_REDIS_HOST'] or process.env['REDIS_HOST'] or "localhost" port: process.env['WEB_REDIS_PORT'] or process.env['REDIS_PORT'] or "6379" password: process.env["WEB_REDIS_PASSWORD"] or process.env["REDIS_PASSWORD"] or "" - cluster: process.env['REDIS_CLUSTER_HOSTS'] internal: realTime: From 08e48afcb99ae50aeaf37b65b698f5293bf94f62 Mon Sep 17 00:00:00 2001 From: Henry Oswald Date: Fri, 25 Jan 2019 10:57:36 +0000 Subject: [PATCH 170/491] more logging --- services/real-time/app.coffee | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/services/real-time/app.coffee b/services/real-time/app.coffee index ca7cacfda3..5a31f98180 100644 --- a/services/real-time/app.coffee +++ b/services/real-time/app.coffee @@ -21,7 +21,8 @@ SessionSockets = require('session.socket.io') CookieParser = require("cookie-parser") -sessionRedisClient.set "hello-a", "hello-there" +sessionRedisClient.set "hello-a", "hello-there", (err)-> + console.log "setting hello-a", err DrainManager = require("./app/js/DrainManager") From 2e46cfb3ebb9062425102327b665c582d3b90a1d Mon Sep 17 00:00:00 2001 From: Henry Oswald Date: Fri, 25 Jan 2019 16:30:16 +0000 Subject: [PATCH 171/491] bump node redis to 1.0.5 --- services/real-time/npm-shrinkwrap.json | 132 ++++++++----------------- services/real-time/package.json | 2 +- 2 files changed, 43 insertions(+), 91 deletions(-) diff --git a/services/real-time/npm-shrinkwrap.json b/services/real-time/npm-shrinkwrap.json index 356b9e51f4..df9b7d673b 100644 --- a/services/real-time/npm-shrinkwrap.json +++ b/services/real-time/npm-shrinkwrap.json @@ -327,11 +327,6 @@ "from": "bintrees@1.0.1", "resolved": "https://registry.npmjs.org/bintrees/-/bintrees-1.0.1.tgz" }, - "bluebird": { - "version": "3.5.1", - "from": "bluebird@>=3.3.4 <4.0.0", - "resolved": "https://registry.npmjs.org/bluebird/-/bluebird-3.5.1.tgz" - }, "body-parser": { "version": "1.18.3", "from": "body-parser@>=1.12.0 <2.0.0", @@ -403,9 +398,9 @@ "resolved": "https://registry.npmjs.org/check-error/-/check-error-1.0.2.tgz" }, "cluster-key-slot": { - "version": "1.0.9", + "version": "1.0.12", "from": "cluster-key-slot@>=1.0.6 <2.0.0", - "resolved": "https://registry.npmjs.org/cluster-key-slot/-/cluster-key-slot-1.0.9.tgz" + "resolved": "https://registry.npmjs.org/cluster-key-slot/-/cluster-key-slot-1.0.12.tgz" }, "coffee-script": { "version": "1.12.4", @@ -530,9 +525,9 @@ "optional": true }, "denque": { - "version": "1.2.3", + "version": "1.4.0", "from": "denque@>=1.1.0 <2.0.0", - "resolved": "https://registry.npmjs.org/denque/-/denque-1.2.3.tgz" + "resolved": "https://registry.npmjs.org/denque/-/denque-1.4.0.tgz" }, "depd": { "version": "1.1.2", @@ -1125,6 +1120,23 @@ "from": "inherits@2.0.3", "resolved": "https://registry.npmjs.org/inherits/-/inherits-2.0.3.tgz" }, + "ioredis": { + "version": "4.6.0", + "from": "ioredis@4.6.0", + "resolved": "https://registry.npmjs.org/ioredis/-/ioredis-4.6.0.tgz", + "dependencies": { + "debug": { + "version": "3.2.6", + "from": "debug@>=3.1.0 <4.0.0", + "resolved": "https://registry.npmjs.org/debug/-/debug-3.2.6.tgz" + }, + "ms": { + "version": "2.1.1", + "from": "ms@>=2.1.1 <3.0.0", + "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.1.tgz" + } + } + }, "ipaddr.js": { "version": "1.6.0", "from": "ipaddr.js@1.6.0", @@ -1202,96 +1214,26 @@ "from": "lodash@>=0.9.2 <0.10.0", "resolved": "https://registry.npmjs.org/lodash/-/lodash-0.9.2.tgz" }, - "lodash.assign": { - "version": "4.2.0", - "from": "lodash.assign@>=4.2.0 <5.0.0", - "resolved": "https://registry.npmjs.org/lodash.assign/-/lodash.assign-4.2.0.tgz" - }, - "lodash.bind": { - "version": "4.2.1", - "from": "lodash.bind@>=4.2.1 <5.0.0", - "resolved": "https://registry.npmjs.org/lodash.bind/-/lodash.bind-4.2.1.tgz" - }, - "lodash.clone": { - "version": "4.5.0", - "from": "lodash.clone@>=4.5.0 <5.0.0", - "resolved": "https://registry.npmjs.org/lodash.clone/-/lodash.clone-4.5.0.tgz" - }, - "lodash.clonedeep": { - "version": "4.5.0", - "from": "lodash.clonedeep@>=4.5.0 <5.0.0", - "resolved": "https://registry.npmjs.org/lodash.clonedeep/-/lodash.clonedeep-4.5.0.tgz" - }, "lodash.defaults": { "version": "4.2.0", "from": "lodash.defaults@>=4.2.0 <5.0.0", "resolved": "https://registry.npmjs.org/lodash.defaults/-/lodash.defaults-4.2.0.tgz" }, - "lodash.difference": { - "version": "4.5.0", - "from": "lodash.difference@>=4.5.0 <5.0.0", - "resolved": "https://registry.npmjs.org/lodash.difference/-/lodash.difference-4.5.0.tgz" - }, "lodash.flatten": { "version": "4.4.0", "from": "lodash.flatten@>=4.4.0 <5.0.0", "resolved": "https://registry.npmjs.org/lodash.flatten/-/lodash.flatten-4.4.0.tgz" }, - "lodash.foreach": { - "version": "4.5.0", - "from": "lodash.foreach@>=4.5.0 <5.0.0", - "resolved": "https://registry.npmjs.org/lodash.foreach/-/lodash.foreach-4.5.0.tgz" - }, "lodash.get": { "version": "4.4.2", "from": "lodash.get@>=4.4.2 <5.0.0", "resolved": "https://registry.npmjs.org/lodash.get/-/lodash.get-4.4.2.tgz" }, - "lodash.isempty": { - "version": "4.4.0", - "from": "lodash.isempty@>=4.4.0 <5.0.0", - "resolved": "https://registry.npmjs.org/lodash.isempty/-/lodash.isempty-4.4.0.tgz" - }, - "lodash.keys": { - "version": "4.2.0", - "from": "lodash.keys@>=4.2.0 <5.0.0", - "resolved": "https://registry.npmjs.org/lodash.keys/-/lodash.keys-4.2.0.tgz" - }, - "lodash.noop": { - "version": "3.0.1", - "from": "lodash.noop@>=3.0.1 <4.0.0", - "resolved": "https://registry.npmjs.org/lodash.noop/-/lodash.noop-3.0.1.tgz" - }, - "lodash.partial": { - "version": "4.2.1", - "from": "lodash.partial@>=4.2.1 <5.0.0", - "resolved": "https://registry.npmjs.org/lodash.partial/-/lodash.partial-4.2.1.tgz" - }, - "lodash.pick": { - "version": "4.4.0", - "from": "lodash.pick@>=4.4.0 <5.0.0", - "resolved": "https://registry.npmjs.org/lodash.pick/-/lodash.pick-4.4.0.tgz" - }, "lodash.pickby": { "version": "4.6.0", "from": "lodash.pickby@>=4.6.0 <5.0.0", "resolved": "https://registry.npmjs.org/lodash.pickby/-/lodash.pickby-4.6.0.tgz" }, - "lodash.sample": { - "version": "4.2.1", - "from": "lodash.sample@>=4.2.1 <5.0.0", - "resolved": "https://registry.npmjs.org/lodash.sample/-/lodash.sample-4.2.1.tgz" - }, - "lodash.shuffle": { - "version": "4.2.0", - "from": "lodash.shuffle@>=4.2.0 <5.0.0", - "resolved": "https://registry.npmjs.org/lodash.shuffle/-/lodash.shuffle-4.2.0.tgz" - }, - "lodash.values": { - "version": "4.3.0", - "from": "lodash.values@>=4.3.0 <5.0.0", - "resolved": "https://registry.npmjs.org/lodash.values/-/lodash.values-4.3.0.tgz" - }, "logger-sharelatex": { "version": "1.5.9", "from": "git+https://github.com/sharelatex/logger-sharelatex.git#v1.5.9", @@ -1728,14 +1670,19 @@ "resolved": "https://registry.npmjs.org/redis/-/redis-0.12.1.tgz" }, "redis-commands": { - "version": "1.3.5", - "from": "redis-commands@>=1.2.0 <2.0.0", - "resolved": "https://registry.npmjs.org/redis-commands/-/redis-commands-1.3.5.tgz" + "version": "1.4.0", + "from": "redis-commands@1.4.0", + "resolved": "https://registry.npmjs.org/redis-commands/-/redis-commands-1.4.0.tgz" + }, + "redis-errors": { + "version": "1.2.0", + "from": "redis-errors@>=1.2.0 <2.0.0", + "resolved": "https://registry.npmjs.org/redis-errors/-/redis-errors-1.2.0.tgz" }, "redis-parser": { - "version": "2.6.0", - "from": "redis-parser@>=2.4.0 <3.0.0", - "resolved": "https://registry.npmjs.org/redis-parser/-/redis-parser-2.6.0.tgz" + "version": "3.0.0", + "from": "redis-parser@>=3.0.0 <4.0.0", + "resolved": "https://registry.npmjs.org/redis-parser/-/redis-parser-3.0.0.tgz" }, "redis-sentinel": { "version": "0.1.1", @@ -1750,9 +1697,9 @@ } }, "redis-sharelatex": { - "version": "4156cfd9ba9444d87a599f6d7035b73b505070f2", - "from": "git+https://github.com/sharelatex/redis-sharelatex.git#4156cfd9ba9444d87a599f6d7035b73b505070f2", - "resolved": "git+https://github.com/sharelatex/redis-sharelatex.git#4156cfd9ba9444d87a599f6d7035b73b505070f2", + "version": "1.0.5", + "from": "git+https://github.com/sharelatex/redis-sharelatex.git#v1.0.5", + "resolved": "git+https://github.com/sharelatex/redis-sharelatex.git#d6130c85bad845359729c45b8467d9d8a3ff44de", "dependencies": { "async": { "version": "2.6.1", @@ -1765,9 +1712,9 @@ "resolved": "https://registry.npmjs.org/coffee-script/-/coffee-script-1.8.0.tgz" }, "lodash": { - "version": "4.17.10", + "version": "4.17.11", "from": "lodash@>=4.17.10 <5.0.0", - "resolved": "https://registry.npmjs.org/lodash/-/lodash-4.17.10.tgz" + "resolved": "https://registry.npmjs.org/lodash/-/lodash-4.17.11.tgz" }, "mkdirp": { "version": "0.3.5", @@ -1962,6 +1909,11 @@ "from": "stack-trace@0.0.9", "resolved": "https://registry.npmjs.org/stack-trace/-/stack-trace-0.0.9.tgz" }, + "standard-as-callback": { + "version": "1.0.1", + "from": "standard-as-callback@>=1.0.0 <2.0.0", + "resolved": "https://registry.npmjs.org/standard-as-callback/-/standard-as-callback-1.0.1.tgz" + }, "statsd-parser": { "version": "0.0.4", "from": "statsd-parser@>=0.0.4 <0.1.0", diff --git a/services/real-time/package.json b/services/real-time/package.json index e5aa8ee54d..6268437247 100644 --- a/services/real-time/package.json +++ b/services/real-time/package.json @@ -30,7 +30,7 @@ "express-session": "^1.9.1", "logger-sharelatex": "git+https://github.com/sharelatex/logger-sharelatex.git#v1.5.9", "metrics-sharelatex": "git+https://github.com/sharelatex/metrics-sharelatex.git#v2.0.12", - "redis-sharelatex": "git+https://github.com/sharelatex/redis-sharelatex.git#4156cfd9ba9444d87a599f6d7035b73b505070f2", + "redis-sharelatex": "git+https://github.com/sharelatex/redis-sharelatex.git#v1.0.5", "request": "~2.34.0", "session.socket.io": "^0.1.6", "settings-sharelatex": "git+https://github.com/sharelatex/settings-sharelatex.git#v1.1.0", From 23e0ce678e3bc0c284a4e148f1175267d284ca22 Mon Sep 17 00:00:00 2001 From: Henry Oswald Date: Thu, 31 Jan 2019 15:33:11 +0000 Subject: [PATCH 172/491] call app real-time --- services/real-time/app.coffee | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/services/real-time/app.coffee b/services/real-time/app.coffee index 5a31f98180..1035f83b68 100644 --- a/services/real-time/app.coffee +++ b/services/real-time/app.coffee @@ -5,7 +5,7 @@ Metrics.initialize(Settings.appName or "real-time") console.log Settings.redis logger = require "logger-sharelatex" -logger.initialize("real-time-sharelatex") +logger.initialize("real-time") Metrics.event_loop.monitor(logger) express = require("express") @@ -20,10 +20,6 @@ RedisStore = require('connect-redis')(session) SessionSockets = require('session.socket.io') CookieParser = require("cookie-parser") - -sessionRedisClient.set "hello-a", "hello-there", (err)-> - console.log "setting hello-a", err - DrainManager = require("./app/js/DrainManager") # Set up socket.io server From 6cdfd5b659e40242d7977b7a4af2df44e3a8fbd4 Mon Sep 17 00:00:00 2001 From: Henry Oswald Date: Fri, 1 Feb 2019 17:25:45 +0000 Subject: [PATCH 173/491] added http auth to config --- services/real-time/Jenkinsfile | 10 +- services/real-time/Makefile | 6 +- services/real-time/buildscript.txt | 5 +- .../real-time/config/settings.defaults.coffee | 4 +- services/real-time/docker-compose.ci.yml | 13 +- services/real-time/docker-compose.yml | 12 +- services/real-time/npm-shrinkwrap.json | 853 +++++++----------- services/real-time/package.json | 8 +- 8 files changed, 349 insertions(+), 562 deletions(-) diff --git a/services/real-time/Jenkinsfile b/services/real-time/Jenkinsfile index c553a7ef79..d7b6b353a1 100644 --- a/services/real-time/Jenkinsfile +++ b/services/real-time/Jenkinsfile @@ -48,8 +48,11 @@ pipeline { } } - stage('Package and publish build') { + stage('Package and docker push') { steps { + sh 'echo ${BUILD_NUMBER} > build_number.txt' + sh 'touch build.tar.gz' // Avoid tar warning about files changing during read + sh 'DOCKER_COMPOSE_FLAGS="-f docker-compose.ci.yml" make tar' withCredentials([file(credentialsId: 'gcr.io_overleaf-ops', variable: 'DOCKER_REPO_KEY_PATH')]) { sh 'docker login -u _json_key --password-stdin https://gcr.io/overleaf-ops < ${DOCKER_REPO_KEY_PATH}' @@ -60,9 +63,12 @@ pipeline { } } - stage('Publish build number') { + stage('Publish to s3') { steps { sh 'echo ${BRANCH_NAME}-${BUILD_NUMBER} > build_number.txt' + withAWS(credentials:'S3_CI_BUILDS_AWS_KEYS', region:"${S3_REGION_BUILD_ARTEFACTS}") { + s3Upload(file:'build.tar.gz', bucket:"${S3_BUCKET_BUILD_ARTEFACTS}", path:"${JOB_NAME}/${BUILD_NUMBER}.tar.gz") + } withAWS(credentials:'S3_CI_BUILDS_AWS_KEYS', region:"${S3_REGION_BUILD_ARTEFACTS}") { // The deployment process uses this file to figure out the latest build s3Upload(file:'build_number.txt', bucket:"${S3_BUCKET_BUILD_ARTEFACTS}", path:"${JOB_NAME}/latest") diff --git a/services/real-time/Makefile b/services/real-time/Makefile index 8620b42f06..fdf938da5a 100644 --- a/services/real-time/Makefile +++ b/services/real-time/Makefile @@ -1,7 +1,7 @@ # This file was auto-generated, do not edit it directly. # Instead run bin/update_build_scripts from # https://github.com/sharelatex/sharelatex-dev-environment -# Version: 1.1.10 +# Version: 1.1.12 BUILD_NUMBER ?= local BRANCH_NAME ?= $(shell git rev-parse --abbrev-ref HEAD) @@ -13,7 +13,6 @@ DOCKER_COMPOSE := BUILD_NUMBER=$(BUILD_NUMBER) \ MOCHA_GREP=${MOCHA_GREP} \ docker-compose ${DOCKER_COMPOSE_FLAGS} - clean: docker rmi ci/$(PROJECT_NAME):$(BRANCH_NAME)-$(BUILD_NUMBER) docker rmi gcr.io/overleaf-ops/$(PROJECT_NAME):$(BRANCH_NAME)-$(BUILD_NUMBER) @@ -40,6 +39,9 @@ build: --tag gcr.io/overleaf-ops/$(PROJECT_NAME):$(BRANCH_NAME)-$(BUILD_NUMBER) \ . +tar: + $(DOCKER_COMPOSE) up tar + publish: docker push $(DOCKER_REPO)/$(PROJECT_NAME):$(BRANCH_NAME)-$(BUILD_NUMBER) diff --git a/services/real-time/buildscript.txt b/services/real-time/buildscript.txt index c343ba8891..5d6ccb471c 100644 --- a/services/real-time/buildscript.txt +++ b/services/real-time/buildscript.txt @@ -1,9 +1,8 @@ ---script-version=1.1.10 real-time +--language=coffeescript --node-version=6.15.1 --acceptance-creds=None ---language=coffeescript --dependencies=mongo,redis --docker-repos=gcr.io/overleaf-ops ---kube=false --build-target=docker +--script-version=1.1.12 diff --git a/services/real-time/config/settings.defaults.coffee b/services/real-time/config/settings.defaults.coffee index 4f48c3bcb3..ed9fa4f90b 100644 --- a/services/real-time/config/settings.defaults.coffee +++ b/services/real-time/config/settings.defaults.coffee @@ -30,8 +30,8 @@ settings = apis: web: url: "http://#{process.env['WEB_HOST'] or "localhost"}:#{process.env['WEB_PORT'] or 3000}" - user: "sharelatex" - pass: "password" + user: process.env['WEB_API_USER'] or "sharelatex" + pass: process.env['WEB_API_PASSWORD'] or "password" documentupdater: url: "http://#{process.env['DOCUMENT_UPDATER_HOST'] or process.env['DOCUPDATER_HOST'] or "localhost"}:3003" diff --git a/services/real-time/docker-compose.ci.yml b/services/real-time/docker-compose.ci.yml index 5ab90e1825..36b52f8f8b 100644 --- a/services/real-time/docker-compose.ci.yml +++ b/services/real-time/docker-compose.ci.yml @@ -1,7 +1,7 @@ # This file was auto-generated, do not edit it directly. # Instead run bin/update_build_scripts from # https://github.com/sharelatex/sharelatex-dev-environment -# Version: 1.1.10 +# Version: 1.1.12 version: "2" @@ -11,6 +11,7 @@ services: user: node command: npm run test:unit:_run + test_acceptance: build: . image: ci/$PROJECT_NAME:$BRANCH_NAME-$BUILD_NUMBER @@ -26,6 +27,16 @@ services: user: node command: npm run test:acceptance:_run + + + tar: + build: . + image: ci/$PROJECT_NAME:$BRANCH_NAME-$BUILD_NUMBER + volumes: + - ./:/tmp/build/ + command: tar -czf /tmp/build/build.tar.gz --exclude=build.tar.gz --exclude-vcs . + user: root + redis: image: redis diff --git a/services/real-time/docker-compose.yml b/services/real-time/docker-compose.yml index aeceafb3f3..8bb7857cb6 100644 --- a/services/real-time/docker-compose.yml +++ b/services/real-time/docker-compose.yml @@ -1,7 +1,7 @@ # This file was auto-generated, do not edit it directly. # Instead run bin/update_build_scripts from # https://github.com/sharelatex/sharelatex-dev-environment -# Version: 1.1.10 +# Version: 1.1.12 version: "2" @@ -33,6 +33,16 @@ services: - redis command: npm run test:acceptance + + + tar: + build: . + image: ci/$PROJECT_NAME:$BRANCH_NAME-$BUILD_NUMBER + volumes: + - ./:/tmp/build/ + command: tar -czf /tmp/build/build.tar.gz --exclude=build.tar.gz --exclude-vcs . + user: root + redis: image: redis diff --git a/services/real-time/npm-shrinkwrap.json b/services/real-time/npm-shrinkwrap.json index df9b7d673b..3706c6be34 100644 --- a/services/real-time/npm-shrinkwrap.json +++ b/services/real-time/npm-shrinkwrap.json @@ -29,11 +29,6 @@ "from": "@google-cloud/common@>=0.26.0 <0.27.0", "resolved": "https://registry.npmjs.org/@google-cloud/common/-/common-0.26.2.tgz" }, - "nan": { - "version": "2.12.1", - "from": "nan@>=2.11.1 <3.0.0", - "resolved": "https://registry.npmjs.org/nan/-/nan-2.12.1.tgz" - }, "through2": { "version": "3.0.0", "from": "through2@>=3.0.0 <4.0.0", @@ -52,14 +47,19 @@ "resolved": "https://registry.npmjs.org/@google-cloud/promisify/-/promisify-0.3.1.tgz" }, "@google-cloud/trace-agent": { - "version": "3.5.0", + "version": "3.5.2", "from": "@google-cloud/trace-agent@>=3.2.0 <4.0.0", - "resolved": "https://registry.npmjs.org/@google-cloud/trace-agent/-/trace-agent-3.5.0.tgz", + "resolved": "https://registry.npmjs.org/@google-cloud/trace-agent/-/trace-agent-3.5.2.tgz", "dependencies": { "@google-cloud/common": { - "version": "0.28.0", - "from": "@google-cloud/common@>=0.28.0 <0.29.0", - "resolved": "https://registry.npmjs.org/@google-cloud/common/-/common-0.28.0.tgz" + "version": "0.30.2", + "from": "@google-cloud/common@>=0.30.0 <0.31.0", + "resolved": "https://registry.npmjs.org/@google-cloud/common/-/common-0.30.2.tgz" + }, + "google-auth-library": { + "version": "3.0.1", + "from": "google-auth-library@>=3.0.0 <4.0.0", + "resolved": "https://registry.npmjs.org/google-auth-library/-/google-auth-library-3.0.1.tgz" }, "uuid": { "version": "3.3.2", @@ -123,28 +123,6 @@ "from": "@sindresorhus/is@>=0.13.0 <0.14.0", "resolved": "https://registry.npmjs.org/@sindresorhus/is/-/is-0.13.0.tgz" }, - "@sinonjs/commons": { - "version": "1.3.0", - "from": "@sinonjs/commons@>=1.2.0 <2.0.0", - "resolved": "https://registry.npmjs.org/@sinonjs/commons/-/commons-1.3.0.tgz", - "dependencies": { - "type-detect": { - "version": "4.0.8", - "from": "type-detect@4.0.8", - "resolved": "https://registry.npmjs.org/type-detect/-/type-detect-4.0.8.tgz" - } - } - }, - "@sinonjs/formatio": { - "version": "3.1.0", - "from": "@sinonjs/formatio@>=3.1.0 <4.0.0", - "resolved": "https://registry.npmjs.org/@sinonjs/formatio/-/formatio-3.1.0.tgz" - }, - "@sinonjs/samsam": { - "version": "3.0.2", - "from": "@sinonjs/samsam@>=3.0.2 <4.0.0", - "resolved": "https://registry.npmjs.org/@sinonjs/samsam/-/samsam-3.0.2.tgz" - }, "@types/caseless": { "version": "0.12.1", "from": "@types/caseless@*", @@ -171,9 +149,9 @@ "resolved": "https://registry.npmjs.org/@types/long/-/long-4.0.0.tgz" }, "@types/node": { - "version": "10.12.18", + "version": "10.12.20", "from": "@types/node@*", - "resolved": "https://registry.npmjs.org/@types/node/-/node-10.12.18.tgz" + "resolved": "https://registry.npmjs.org/@types/node/-/node-10.12.20.tgz" }, "@types/request": { "version": "2.48.1", @@ -186,14 +164,9 @@ "resolved": "https://registry.npmjs.org/@types/semver/-/semver-5.5.0.tgz" }, "@types/tough-cookie": { - "version": "2.3.4", + "version": "2.3.5", "from": "@types/tough-cookie@*", - "resolved": "https://registry.npmjs.org/@types/tough-cookie/-/tough-cookie-2.3.4.tgz" - }, - "abbrev": { - "version": "1.1.1", - "from": "abbrev@>=1.0.0 <2.0.0", - "resolved": "https://registry.npmjs.org/abbrev/-/abbrev-1.1.1.tgz" + "resolved": "https://registry.npmjs.org/@types/tough-cookie/-/tough-cookie-2.3.5.tgz" }, "accepts": { "version": "1.3.5", @@ -215,38 +188,16 @@ "from": "agent-base@>=4.1.0 <5.0.0", "resolved": "https://registry.npmjs.org/agent-base/-/agent-base-4.2.1.tgz" }, - "ansi-regex": { - "version": "0.2.1", - "from": "ansi-regex@>=0.2.0 <0.3.0", - "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-0.2.1.tgz" - }, - "ansi-styles": { - "version": "1.1.0", - "from": "ansi-styles@>=1.1.0 <2.0.0", - "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-1.1.0.tgz" - }, - "argparse": { - "version": "0.1.16", - "from": "argparse@>=0.1.11 <0.2.0", - "resolved": "https://registry.npmjs.org/argparse/-/argparse-0.1.16.tgz", - "dependencies": { - "underscore.string": { - "version": "2.4.0", - "from": "underscore.string@>=2.4.0 <2.5.0", - "resolved": "https://registry.npmjs.org/underscore.string/-/underscore.string-2.4.0.tgz" - } - } + "ajv": { + "version": "6.7.0", + "from": "ajv@>=6.5.5 <7.0.0", + "resolved": "https://registry.npmjs.org/ajv/-/ajv-6.7.0.tgz" }, "array-flatten": { "version": "1.1.1", "from": "array-flatten@1.1.1", "resolved": "https://registry.npmjs.org/array-flatten/-/array-flatten-1.1.1.tgz" }, - "array-from": { - "version": "2.1.1", - "from": "array-from@>=2.1.1 <3.0.0", - "resolved": "https://registry.npmjs.org/array-from/-/array-from-2.1.1.tgz" - }, "arrify": { "version": "1.0.1", "from": "arrify@>=1.0.1 <2.0.0", @@ -280,12 +231,22 @@ "from": "async-listener@>=0.6.0 <0.7.0", "resolved": "https://registry.npmjs.org/async-listener/-/async-listener-0.6.10.tgz" }, + "asynckit": { + "version": "0.4.0", + "from": "asynckit@>=0.4.0 <0.5.0", + "resolved": "https://registry.npmjs.org/asynckit/-/asynckit-0.4.0.tgz" + }, "aws-sign2": { "version": "0.5.0", "from": "aws-sign2@>=0.5.0 <0.6.0", "resolved": "https://registry.npmjs.org/aws-sign2/-/aws-sign2-0.5.0.tgz", "optional": true }, + "aws4": { + "version": "1.8.0", + "from": "aws4@>=1.8.0 <2.0.0", + "resolved": "https://registry.npmjs.org/aws4/-/aws4-1.8.0.tgz" + }, "axios": { "version": "0.18.0", "from": "axios@>=0.18.0 <0.19.0", @@ -296,6 +257,11 @@ "from": "balanced-match@^1.0.0", "resolved": "https://registry.npmjs.org/balanced-match/-/balanced-match-1.0.0.tgz" }, + "base64-js": { + "version": "1.3.0", + "from": "base64-js@>=1.3.0 <2.0.0", + "resolved": "https://registry.npmjs.org/base64-js/-/base64-js-1.3.0.tgz" + }, "base64-url": { "version": "1.2.1", "from": "base64-url@1.2.1", @@ -312,15 +278,20 @@ "from": "basic-auth-connect@>=1.0.0 <2.0.0", "resolved": "https://registry.npmjs.org/basic-auth-connect/-/basic-auth-connect-1.0.0.tgz" }, + "bcrypt-pbkdf": { + "version": "1.0.2", + "from": "bcrypt-pbkdf@>=1.0.0 <2.0.0", + "resolved": "https://registry.npmjs.org/bcrypt-pbkdf/-/bcrypt-pbkdf-1.0.2.tgz" + }, "bignumber.js": { "version": "7.2.1", "from": "bignumber.js@>=7.0.0 <8.0.0", "resolved": "https://registry.npmjs.org/bignumber.js/-/bignumber.js-7.2.1.tgz" }, "bindings": { - "version": "1.3.1", + "version": "1.4.0", "from": "bindings@>=1.2.1 <2.0.0", - "resolved": "https://registry.npmjs.org/bindings/-/bindings-1.3.1.tgz" + "resolved": "https://registry.npmjs.org/bindings/-/bindings-1.4.0.tgz" }, "bintrees": { "version": "1.0.1", @@ -381,22 +352,17 @@ "from": "bytes@3.0.0", "resolved": "https://registry.npmjs.org/bytes/-/bytes-3.0.0.tgz" }, + "caseless": { + "version": "0.12.0", + "from": "caseless@>=0.12.0 <0.13.0", + "resolved": "https://registry.npmjs.org/caseless/-/caseless-0.12.0.tgz" + }, "chai": { "version": "1.9.2", "from": "chai@>=1.9.1 <1.10.0", "resolved": "https://registry.npmjs.org/chai/-/chai-1.9.2.tgz", "dev": true }, - "chalk": { - "version": "0.5.1", - "from": "chalk@>=0.5.0 <0.6.0", - "resolved": "https://registry.npmjs.org/chalk/-/chalk-0.5.1.tgz" - }, - "check-error": { - "version": "1.0.2", - "from": "check-error@>=1.0.1 <2.0.0", - "resolved": "https://registry.npmjs.org/check-error/-/check-error-1.0.2.tgz" - }, "cluster-key-slot": { "version": "1.0.12", "from": "cluster-key-slot@>=1.0.6 <2.0.0", @@ -407,22 +373,12 @@ "from": "coffee-script@1.12.4", "resolved": "https://registry.npmjs.org/coffee-script/-/coffee-script-1.12.4.tgz" }, - "colors": { - "version": "0.6.2", - "from": "colors@>=0.6.2 <0.7.0", - "resolved": "https://registry.npmjs.org/colors/-/colors-0.6.2.tgz" - }, "combined-stream": { "version": "0.0.7", "from": "combined-stream@>=0.0.4 <0.1.0", "resolved": "https://registry.npmjs.org/combined-stream/-/combined-stream-0.0.7.tgz", "optional": true }, - "commander": { - "version": "2.0.0", - "from": "commander@2.0.0", - "resolved": "http://registry.npmjs.org/commander/-/commander-2.0.0.tgz" - }, "concat-map": { "version": "0.0.1", "from": "concat-map@0.0.1", @@ -497,10 +453,17 @@ "resolved": "https://registry.npmjs.org/ctype/-/ctype-0.5.3.tgz", "optional": true }, - "dateformat": { - "version": "1.0.2-1.2.3", - "from": "dateformat@1.0.2-1.2.3", - "resolved": "https://registry.npmjs.org/dateformat/-/dateformat-1.0.2-1.2.3.tgz" + "dashdash": { + "version": "1.14.1", + "from": "dashdash@>=1.12.0 <2.0.0", + "resolved": "https://registry.npmjs.org/dashdash/-/dashdash-1.14.1.tgz", + "dependencies": { + "assert-plus": { + "version": "1.0.0", + "from": "assert-plus@^1.0.0", + "resolved": "https://registry.npmjs.org/assert-plus/-/assert-plus-1.0.0.tgz" + } + } }, "debug": { "version": "2.6.9", @@ -539,11 +502,6 @@ "from": "destroy@>=1.0.4 <1.1.0", "resolved": "https://registry.npmjs.org/destroy/-/destroy-1.0.4.tgz" }, - "diff": { - "version": "1.0.7", - "from": "diff@1.0.7", - "resolved": "https://registry.npmjs.org/diff/-/diff-1.0.7.tgz" - }, "dtrace-provider": { "version": "0.2.8", "from": "dtrace-provider@0.2.8", @@ -556,6 +514,11 @@ "from": "duplexify@>=3.6.0 <4.0.0", "resolved": "https://registry.npmjs.org/duplexify/-/duplexify-3.6.1.tgz" }, + "ecc-jsbn": { + "version": "0.1.2", + "from": "ecc-jsbn@>=0.1.1 <0.2.0", + "resolved": "https://registry.npmjs.org/ecc-jsbn/-/ecc-jsbn-0.1.2.tgz" + }, "ecdsa-sig-formatter": { "version": "1.0.10", "from": "ecdsa-sig-formatter@1.0.10", @@ -604,28 +567,14 @@ "escape-string-regexp": { "version": "1.0.5", "from": "escape-string-regexp@>=1.0.0 <2.0.0", - "resolved": "https://registry.npmjs.org/escape-string-regexp/-/escape-string-regexp-1.0.5.tgz" - }, - "esprima": { - "version": "1.0.4", - "from": "esprima@>=1.0.2 <1.1.0", - "resolved": "https://registry.npmjs.org/esprima/-/esprima-1.0.4.tgz" + "resolved": "https://registry.npmjs.org/escape-string-regexp/-/escape-string-regexp-1.0.5.tgz", + "dev": true }, "etag": { "version": "1.8.1", "from": "etag@>=1.8.1 <1.9.0", "resolved": "https://registry.npmjs.org/etag/-/etag-1.8.1.tgz" }, - "eventemitter2": { - "version": "0.4.14", - "from": "eventemitter2@>=0.4.13 <0.5.0", - "resolved": "https://registry.npmjs.org/eventemitter2/-/eventemitter2-0.4.14.tgz" - }, - "exit": { - "version": "0.1.2", - "from": "exit@>=0.1.1 <0.2.0", - "resolved": "https://registry.npmjs.org/exit/-/exit-0.1.2.tgz" - }, "express": { "version": "4.16.3", "from": "express@>=4.10.1 <5.0.0", @@ -692,6 +641,31 @@ "from": "extend@>=3.0.1 <4.0.0", "resolved": "https://registry.npmjs.org/extend/-/extend-3.0.2.tgz" }, + "extsprintf": { + "version": "1.3.0", + "from": "extsprintf@1.3.0", + "resolved": "https://registry.npmjs.org/extsprintf/-/extsprintf-1.3.0.tgz" + }, + "fast-deep-equal": { + "version": "2.0.1", + "from": "fast-deep-equal@>=2.0.1 <3.0.0", + "resolved": "https://registry.npmjs.org/fast-deep-equal/-/fast-deep-equal-2.0.1.tgz" + }, + "fast-json-stable-stringify": { + "version": "2.0.0", + "from": "fast-json-stable-stringify@>=2.0.0 <3.0.0", + "resolved": "https://registry.npmjs.org/fast-json-stable-stringify/-/fast-json-stable-stringify-2.0.0.tgz" + }, + "fast-text-encoding": { + "version": "1.0.0", + "from": "fast-text-encoding@>=1.0.0 <2.0.0", + "resolved": "https://registry.npmjs.org/fast-text-encoding/-/fast-text-encoding-1.0.0.tgz" + }, + "file-uri-to-path": { + "version": "1.0.0", + "from": "file-uri-to-path@1.0.0", + "resolved": "https://registry.npmjs.org/file-uri-to-path/-/file-uri-to-path-1.0.0.tgz" + }, "finalhandler": { "version": "1.1.1", "from": "finalhandler@1.1.1", @@ -709,28 +683,6 @@ "from": "findit2@>=2.2.3 <3.0.0", "resolved": "https://registry.npmjs.org/findit2/-/findit2-2.2.3.tgz" }, - "findup-sync": { - "version": "0.1.3", - "from": "findup-sync@>=0.1.2 <0.2.0", - "resolved": "https://registry.npmjs.org/findup-sync/-/findup-sync-0.1.3.tgz", - "dependencies": { - "glob": { - "version": "3.2.11", - "from": "glob@>=3.2.9 <3.3.0", - "resolved": "https://registry.npmjs.org/glob/-/glob-3.2.11.tgz" - }, - "lodash": { - "version": "2.4.2", - "from": "lodash@>=2.4.1 <2.5.0", - "resolved": "https://registry.npmjs.org/lodash/-/lodash-2.4.2.tgz" - }, - "minimatch": { - "version": "0.3.0", - "from": "minimatch@>=0.3.0 <0.4.0", - "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-0.3.0.tgz" - } - } - }, "flexbuffer": { "version": "0.0.6", "from": "flexbuffer@0.0.6", @@ -777,18 +729,6 @@ "from": "fresh@0.5.2", "resolved": "https://registry.npmjs.org/fresh/-/fresh-0.5.2.tgz" }, - "fs-extra": { - "version": "0.9.1", - "from": "fs-extra@>=0.9.1 <0.10.0", - "resolved": "https://registry.npmjs.org/fs-extra/-/fs-extra-0.9.1.tgz", - "dependencies": { - "ncp": { - "version": "0.5.1", - "from": "ncp@>=0.5.1 <0.6.0", - "resolved": "https://registry.npmjs.org/ncp/-/ncp-0.5.1.tgz" - } - } - }, "fs.realpath": { "version": "1.0.0", "from": "fs.realpath@>=1.0.0 <2.0.0", @@ -796,29 +736,32 @@ "dev": true }, "gaxios": { - "version": "1.0.4", - "from": "gaxios@>=1.0.2 <2.0.0", - "resolved": "https://registry.npmjs.org/gaxios/-/gaxios-1.0.4.tgz" + "version": "1.2.7", + "from": "gaxios@>=1.0.4 <2.0.0", + "resolved": "https://registry.npmjs.org/gaxios/-/gaxios-1.2.7.tgz" }, "gcp-metadata": { "version": "0.9.3", "from": "gcp-metadata@>=0.9.0 <0.10.0", "resolved": "https://registry.npmjs.org/gcp-metadata/-/gcp-metadata-0.9.3.tgz" }, - "get-func-name": { - "version": "2.0.0", - "from": "get-func-name@>=2.0.0 <3.0.0", - "resolved": "https://registry.npmjs.org/get-func-name/-/get-func-name-2.0.0.tgz" - }, - "getobject": { - "version": "0.1.0", - "from": "getobject@>=0.1.0 <0.2.0", - "resolved": "https://registry.npmjs.org/getobject/-/getobject-0.1.0.tgz" + "getpass": { + "version": "0.1.7", + "from": "getpass@>=0.1.1 <0.2.0", + "resolved": "https://registry.npmjs.org/getpass/-/getpass-0.1.7.tgz", + "dependencies": { + "assert-plus": { + "version": "1.0.0", + "from": "assert-plus@^1.0.0", + "resolved": "https://registry.npmjs.org/assert-plus/-/assert-plus-1.0.0.tgz" + } + } }, "glob": { "version": "6.0.4", "from": "glob@^6.0.1", - "resolved": "https://registry.npmjs.org/glob/-/glob-6.0.4.tgz" + "resolved": "https://registry.npmjs.org/glob/-/glob-6.0.4.tgz", + "optional": true }, "google-auth-library": { "version": "2.0.2", @@ -829,11 +772,6 @@ "version": "0.7.0", "from": "gcp-metadata@>=0.7.0 <0.8.0", "resolved": "https://registry.npmjs.org/gcp-metadata/-/gcp-metadata-0.7.0.tgz" - }, - "lru-cache": { - "version": "5.1.1", - "from": "lru-cache@>=5.0.0 <6.0.0", - "resolved": "https://registry.npmjs.org/lru-cache/-/lru-cache-5.1.1.tgz" } } }, @@ -842,213 +780,27 @@ "from": "google-p12-pem@>=1.0.0 <2.0.0", "resolved": "https://registry.npmjs.org/google-p12-pem/-/google-p12-pem-1.0.3.tgz" }, - "graceful-fs": { - "version": "1.2.3", - "from": "graceful-fs@>=1.2.0 <1.3.0", - "resolved": "https://registry.npmjs.org/graceful-fs/-/graceful-fs-1.2.3.tgz" - }, - "growl": { - "version": "1.7.0", - "from": "growl@>=1.7.0 <1.8.0", - "resolved": "https://registry.npmjs.org/growl/-/growl-1.7.0.tgz" - }, - "grunt": { - "version": "0.4.5", - "from": "grunt@>=0.4.5 <0.5.0", - "resolved": "https://registry.npmjs.org/grunt/-/grunt-0.4.5.tgz", - "dependencies": { - "async": { - "version": "0.1.22", - "from": "async@>=0.1.22 <0.2.0", - "resolved": "https://registry.npmjs.org/async/-/async-0.1.22.tgz" - }, - "coffee-script": { - "version": "1.3.3", - "from": "coffee-script@>=1.3.3 <1.4.0", - "resolved": "https://registry.npmjs.org/coffee-script/-/coffee-script-1.3.3.tgz" - }, - "glob": { - "version": "3.1.21", - "from": "glob@>=3.1.21 <3.2.0", - "resolved": "https://registry.npmjs.org/glob/-/glob-3.1.21.tgz" - }, - "iconv-lite": { - "version": "0.2.11", - "from": "iconv-lite@>=0.2.11 <0.3.0", - "resolved": "https://registry.npmjs.org/iconv-lite/-/iconv-lite-0.2.11.tgz" - }, - "inherits": { - "version": "1.0.2", - "from": "inherits@>=1.0.0 <2.0.0", - "resolved": "https://registry.npmjs.org/inherits/-/inherits-1.0.2.tgz" - }, - "minimatch": { - "version": "0.2.14", - "from": "minimatch@>=0.2.12 <0.3.0", - "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-0.2.14.tgz" - }, - "rimraf": { - "version": "2.2.8", - "from": "rimraf@>=2.2.8 <2.3.0", - "resolved": "https://registry.npmjs.org/rimraf/-/rimraf-2.2.8.tgz" - } - } - }, - "grunt-bunyan": { - "version": "0.5.0", - "from": "grunt-bunyan@>=0.5.0 <0.6.0", - "resolved": "https://registry.npmjs.org/grunt-bunyan/-/grunt-bunyan-0.5.0.tgz", - "dependencies": { - "lodash": { - "version": "2.4.2", - "from": "lodash@>=2.4.1 <2.5.0", - "resolved": "https://registry.npmjs.org/lodash/-/lodash-2.4.2.tgz" - } - } - }, - "grunt-contrib-clean": { - "version": "0.6.0", - "from": "grunt-contrib-clean@>=0.6.0 <0.7.0", - "resolved": "https://registry.npmjs.org/grunt-contrib-clean/-/grunt-contrib-clean-0.6.0.tgz", - "dependencies": { - "rimraf": { - "version": "2.2.8", - "from": "rimraf@>=2.2.1 <2.3.0", - "resolved": "https://registry.npmjs.org/rimraf/-/rimraf-2.2.8.tgz" - } - } - }, - "grunt-contrib-coffee": { - "version": "0.11.1", - "from": "grunt-contrib-coffee@>=0.11.0 <0.12.0", - "resolved": "https://registry.npmjs.org/grunt-contrib-coffee/-/grunt-contrib-coffee-0.11.1.tgz", - "dependencies": { - "coffee-script": { - "version": "1.7.1", - "from": "coffee-script@>=1.7.0 <1.8.0", - "resolved": "https://registry.npmjs.org/coffee-script/-/coffee-script-1.7.1.tgz" - }, - "lodash": { - "version": "2.4.2", - "from": "lodash@>=2.4.1 <2.5.0", - "resolved": "https://registry.npmjs.org/lodash/-/lodash-2.4.2.tgz" - }, - "mkdirp": { - "version": "0.3.5", - "from": "mkdirp@>=0.3.5 <0.4.0", - "resolved": "https://registry.npmjs.org/mkdirp/-/mkdirp-0.3.5.tgz" - } - } - }, - "grunt-execute": { - "version": "0.2.2", - "from": "grunt-execute@>=0.2.2 <0.3.0", - "resolved": "https://registry.npmjs.org/grunt-execute/-/grunt-execute-0.2.2.tgz" - }, - "grunt-legacy-log": { - "version": "0.1.3", - "from": "grunt-legacy-log@>=0.1.0 <0.2.0", - "resolved": "https://registry.npmjs.org/grunt-legacy-log/-/grunt-legacy-log-0.1.3.tgz", - "dependencies": { - "lodash": { - "version": "2.4.2", - "from": "lodash@>=2.4.1 <2.5.0", - "resolved": "https://registry.npmjs.org/lodash/-/lodash-2.4.2.tgz" - }, - "underscore.string": { - "version": "2.3.3", - "from": "underscore.string@>=2.3.3 <2.4.0", - "resolved": "https://registry.npmjs.org/underscore.string/-/underscore.string-2.3.3.tgz" - } - } - }, - "grunt-legacy-log-utils": { - "version": "0.1.1", - "from": "grunt-legacy-log-utils@>=0.1.1 <0.2.0", - "resolved": "https://registry.npmjs.org/grunt-legacy-log-utils/-/grunt-legacy-log-utils-0.1.1.tgz", - "dependencies": { - "lodash": { - "version": "2.4.2", - "from": "lodash@>=2.4.1 <2.5.0", - "resolved": "https://registry.npmjs.org/lodash/-/lodash-2.4.2.tgz" - }, - "underscore.string": { - "version": "2.3.3", - "from": "underscore.string@>=2.3.3 <2.4.0", - "resolved": "https://registry.npmjs.org/underscore.string/-/underscore.string-2.3.3.tgz" - } - } - }, - "grunt-legacy-util": { - "version": "0.2.0", - "from": "grunt-legacy-util@>=0.2.0 <0.3.0", - "resolved": "https://registry.npmjs.org/grunt-legacy-util/-/grunt-legacy-util-0.2.0.tgz", - "dependencies": { - "async": { - "version": "0.1.22", - "from": "async@>=0.1.22 <0.2.0", - "resolved": "https://registry.npmjs.org/async/-/async-0.1.22.tgz" - } - } - }, - "grunt-mocha-test": { - "version": "0.11.0", - "from": "grunt-mocha-test@>=0.11.0 <0.12.0", - "resolved": "https://registry.npmjs.org/grunt-mocha-test/-/grunt-mocha-test-0.11.0.tgz", - "dependencies": { - "glob": { - "version": "3.2.3", - "from": "glob@3.2.3", - "resolved": "https://registry.npmjs.org/glob/-/glob-3.2.3.tgz" - }, - "graceful-fs": { - "version": "2.0.3", - "from": "graceful-fs@>=2.0.0 <2.1.0", - "resolved": "https://registry.npmjs.org/graceful-fs/-/graceful-fs-2.0.3.tgz" - }, - "minimatch": { - "version": "0.2.14", - "from": "minimatch@>=0.2.11 <0.3.0", - "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-0.2.14.tgz" - }, - "mkdirp": { - "version": "0.3.5", - "from": "mkdirp@0.3.5", - "resolved": "http://registry.npmjs.org/mkdirp/-/mkdirp-0.3.5.tgz" - }, - "mocha": { - "version": "1.20.1", - "from": "mocha@>=1.20.0 <1.21.0", - "resolved": "https://registry.npmjs.org/mocha/-/mocha-1.20.1.tgz" - } - } - }, "gtoken": { - "version": "2.3.0", + "version": "2.3.2", "from": "gtoken@>=2.3.0 <3.0.0", - "resolved": "https://registry.npmjs.org/gtoken/-/gtoken-2.3.0.tgz", + "resolved": "https://registry.npmjs.org/gtoken/-/gtoken-2.3.2.tgz", "dependencies": { "mime": { "version": "2.4.0", "from": "mime@>=2.2.0 <3.0.0", "resolved": "https://registry.npmjs.org/mime/-/mime-2.4.0.tgz" - }, - "pify": { - "version": "3.0.0", - "from": "pify@>=3.0.0 <4.0.0", - "resolved": "https://registry.npmjs.org/pify/-/pify-3.0.0.tgz" } } }, - "has-ansi": { - "version": "0.1.0", - "from": "has-ansi@>=0.1.0 <0.2.0", - "resolved": "https://registry.npmjs.org/has-ansi/-/has-ansi-0.1.0.tgz" + "har-schema": { + "version": "2.0.0", + "from": "har-schema@>=2.0.0 <3.0.0", + "resolved": "https://registry.npmjs.org/har-schema/-/har-schema-2.0.0.tgz" }, - "has-flag": { - "version": "3.0.0", - "from": "has-flag@>=3.0.0 <4.0.0", - "resolved": "https://registry.npmjs.org/has-flag/-/has-flag-3.0.0.tgz" + "har-validator": { + "version": "5.1.3", + "from": "har-validator@>=5.1.0 <5.2.0", + "resolved": "https://registry.npmjs.org/har-validator/-/har-validator-5.1.3.tgz" }, "hawk": { "version": "1.0.0", @@ -1063,20 +815,15 @@ "dev": true }, "hex2dec": { - "version": "1.1.1", + "version": "1.1.2", "from": "hex2dec@>=1.0.1 <2.0.0", - "resolved": "https://registry.npmjs.org/hex2dec/-/hex2dec-1.1.1.tgz" + "resolved": "https://registry.npmjs.org/hex2dec/-/hex2dec-1.1.2.tgz" }, "hoek": { "version": "0.9.1", "from": "hoek@>=0.9.0 <0.10.0", "resolved": "https://registry.npmjs.org/hoek/-/hoek-0.9.1.tgz" }, - "hooker": { - "version": "0.2.3", - "from": "hooker@>=0.2.3 <0.3.0", - "resolved": "https://registry.npmjs.org/hooker/-/hooker-0.2.3.tgz" - }, "http-errors": { "version": "1.6.3", "from": "http-errors@>=1.6.3 <1.7.0", @@ -1152,67 +899,72 @@ "from": "is-buffer@>=1.1.5 <2.0.0", "resolved": "https://registry.npmjs.org/is-buffer/-/is-buffer-1.1.6.tgz" }, + "is-typedarray": { + "version": "1.0.0", + "from": "is-typedarray@>=1.0.0 <1.1.0", + "resolved": "https://registry.npmjs.org/is-typedarray/-/is-typedarray-1.0.0.tgz" + }, "isarray": { - "version": "0.0.1", - "from": "isarray@0.0.1", - "resolved": "https://registry.npmjs.org/isarray/-/isarray-0.0.1.tgz" + "version": "1.0.0", + "from": "isarray@>=1.0.0 <1.1.0", + "resolved": "https://registry.npmjs.org/isarray/-/isarray-1.0.0.tgz" }, - "jade": { - "version": "0.26.3", - "from": "jade@0.26.3", - "resolved": "https://registry.npmjs.org/jade/-/jade-0.26.3.tgz", - "dependencies": { - "commander": { - "version": "0.6.1", - "from": "commander@0.6.1", - "resolved": "https://registry.npmjs.org/commander/-/commander-0.6.1.tgz" - }, - "mkdirp": { - "version": "0.3.0", - "from": "mkdirp@0.3.0", - "resolved": "https://registry.npmjs.org/mkdirp/-/mkdirp-0.3.0.tgz" - } - } + "isstream": { + "version": "0.1.2", + "from": "isstream@>=0.1.2 <0.2.0", + "resolved": "https://registry.npmjs.org/isstream/-/isstream-0.1.2.tgz" }, - "js-yaml": { - "version": "2.0.5", - "from": "js-yaml@>=2.0.5 <2.1.0", - "resolved": "https://registry.npmjs.org/js-yaml/-/js-yaml-2.0.5.tgz" + "jsbn": { + "version": "0.1.1", + "from": "jsbn@>=0.1.0 <0.2.0", + "resolved": "https://registry.npmjs.org/jsbn/-/jsbn-0.1.1.tgz" }, "json-bigint": { "version": "0.3.0", "from": "json-bigint@>=0.3.0 <0.4.0", "resolved": "https://registry.npmjs.org/json-bigint/-/json-bigint-0.3.0.tgz" }, + "json-schema": { + "version": "0.2.3", + "from": "json-schema@0.2.3", + "resolved": "https://registry.npmjs.org/json-schema/-/json-schema-0.2.3.tgz" + }, + "json-schema-traverse": { + "version": "0.4.1", + "from": "json-schema-traverse@>=0.4.1 <0.5.0", + "resolved": "https://registry.npmjs.org/json-schema-traverse/-/json-schema-traverse-0.4.1.tgz" + }, "json-stringify-safe": { "version": "5.0.1", "from": "json-stringify-safe@5.0.1", "resolved": "https://registry.npmjs.org/json-stringify-safe/-/json-stringify-safe-5.0.1.tgz" }, - "jsonfile": { - "version": "1.1.1", - "from": "jsonfile@~1.1.0", - "resolved": "https://registry.npmjs.org/jsonfile/-/jsonfile-1.1.1.tgz" - }, - "just-extend": { - "version": "4.0.2", - "from": "just-extend@>=4.0.2 <5.0.0", - "resolved": "https://registry.npmjs.org/just-extend/-/just-extend-4.0.2.tgz" + "jsprim": { + "version": "1.4.1", + "from": "jsprim@>=1.2.2 <2.0.0", + "resolved": "https://registry.npmjs.org/jsprim/-/jsprim-1.4.1.tgz", + "dependencies": { + "assert-plus": { + "version": "1.0.0", + "from": "assert-plus@1.0.0", + "resolved": "https://registry.npmjs.org/assert-plus/-/assert-plus-1.0.0.tgz" + } + } }, "jwa": { - "version": "1.1.6", - "from": "jwa@>=1.1.5 <2.0.0", - "resolved": "https://registry.npmjs.org/jwa/-/jwa-1.1.6.tgz" + "version": "1.2.0", + "from": "jwa@>=1.2.0 <2.0.0", + "resolved": "https://registry.npmjs.org/jwa/-/jwa-1.2.0.tgz" }, "jws": { - "version": "3.1.5", + "version": "3.2.1", "from": "jws@>=3.1.5 <4.0.0", - "resolved": "https://registry.npmjs.org/jws/-/jws-3.1.5.tgz" + "resolved": "https://registry.npmjs.org/jws/-/jws-3.2.1.tgz" }, "lodash": { - "version": "0.9.2", - "from": "lodash@>=0.9.2 <0.10.0", - "resolved": "https://registry.npmjs.org/lodash/-/lodash-0.9.2.tgz" + "version": "4.17.11", + "from": "lodash@>=4.17.10 <5.0.0", + "resolved": "https://registry.npmjs.org/lodash/-/lodash-4.17.11.tgz" }, "lodash.defaults": { "version": "4.2.0", @@ -1224,45 +976,40 @@ "from": "lodash.flatten@>=4.4.0 <5.0.0", "resolved": "https://registry.npmjs.org/lodash.flatten/-/lodash.flatten-4.4.0.tgz" }, - "lodash.get": { - "version": "4.4.2", - "from": "lodash.get@>=4.4.2 <5.0.0", - "resolved": "https://registry.npmjs.org/lodash.get/-/lodash.get-4.4.2.tgz" - }, "lodash.pickby": { "version": "4.6.0", "from": "lodash.pickby@>=4.6.0 <5.0.0", "resolved": "https://registry.npmjs.org/lodash.pickby/-/lodash.pickby-4.6.0.tgz" }, "logger-sharelatex": { - "version": "1.5.9", - "from": "git+https://github.com/sharelatex/logger-sharelatex.git#v1.5.9", - "resolved": "git+https://github.com/sharelatex/logger-sharelatex.git#e8e1b95052f62e107336053e4a983f81cdbdf589", + "version": "1.6.0", + "from": "logger-sharelatex@1.6.0", + "resolved": "https://registry.npmjs.org/logger-sharelatex/-/logger-sharelatex-1.6.0.tgz", "dependencies": { - "assertion-error": { - "version": "1.1.0", - "from": "assertion-error@>=1.0.1 <2.0.0", - "resolved": "https://registry.npmjs.org/assertion-error/-/assertion-error-1.1.0.tgz" + "assert-plus": { + "version": "1.0.0", + "from": "assert-plus@>=1.0.0 <2.0.0", + "resolved": "https://registry.npmjs.org/assert-plus/-/assert-plus-1.0.0.tgz" + }, + "aws-sign2": { + "version": "0.7.0", + "from": "aws-sign2@>=0.7.0 <0.8.0", + "resolved": "https://registry.npmjs.org/aws-sign2/-/aws-sign2-0.7.0.tgz" }, "bunyan": { "version": "1.5.1", "from": "bunyan@1.5.1", "resolved": "https://registry.npmjs.org/bunyan/-/bunyan-1.5.1.tgz" }, - "chai": { - "version": "4.2.0", - "from": "chai@latest", - "resolved": "https://registry.npmjs.org/chai/-/chai-4.2.0.tgz" + "combined-stream": { + "version": "1.0.7", + "from": "combined-stream@>=1.0.6 <1.1.0", + "resolved": "https://registry.npmjs.org/combined-stream/-/combined-stream-1.0.7.tgz" }, - "deep-eql": { - "version": "3.0.1", - "from": "deep-eql@>=3.0.0 <4.0.0", - "resolved": "https://registry.npmjs.org/deep-eql/-/deep-eql-3.0.1.tgz" - }, - "diff": { - "version": "3.5.0", - "from": "diff@>=3.5.0 <4.0.0", - "resolved": "https://registry.npmjs.org/diff/-/diff-3.5.0.tgz" + "delayed-stream": { + "version": "1.0.0", + "from": "delayed-stream@>=1.0.0 <1.1.0", + "resolved": "https://registry.npmjs.org/delayed-stream/-/delayed-stream-1.0.0.tgz" }, "dtrace-provider": { "version": "0.6.0", @@ -1270,47 +1017,72 @@ "resolved": "https://registry.npmjs.org/dtrace-provider/-/dtrace-provider-0.6.0.tgz", "optional": true }, - "sandboxed-module": { - "version": "2.0.3", - "from": "sandboxed-module@latest", - "resolved": "https://registry.npmjs.org/sandboxed-module/-/sandboxed-module-2.0.3.tgz" + "forever-agent": { + "version": "0.6.1", + "from": "forever-agent@>=0.6.1 <0.7.0", + "resolved": "https://registry.npmjs.org/forever-agent/-/forever-agent-0.6.1.tgz" }, - "sinon": { - "version": "7.2.2", - "from": "sinon@latest", - "resolved": "https://registry.npmjs.org/sinon/-/sinon-7.2.2.tgz" + "form-data": { + "version": "2.3.3", + "from": "form-data@>=2.3.2 <2.4.0", + "resolved": "https://registry.npmjs.org/form-data/-/form-data-2.3.3.tgz" }, - "supports-color": { - "version": "5.5.0", - "from": "supports-color@>=5.5.0 <6.0.0", - "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-5.5.0.tgz" + "http-signature": { + "version": "1.2.0", + "from": "http-signature@>=1.2.0 <1.3.0", + "resolved": "https://registry.npmjs.org/http-signature/-/http-signature-1.2.0.tgz" }, - "timekeeper": { - "version": "1.0.0", - "from": "timekeeper@>=1.0.0 <2.0.0", - "resolved": "https://registry.npmjs.org/timekeeper/-/timekeeper-1.0.0.tgz" + "mime-db": { + "version": "1.37.0", + "from": "mime-db@>=1.37.0 <1.38.0", + "resolved": "https://registry.npmjs.org/mime-db/-/mime-db-1.37.0.tgz" }, - "type-detect": { - "version": "4.0.8", - "from": "type-detect@>=4.0.0 <5.0.0", - "resolved": "https://registry.npmjs.org/type-detect/-/type-detect-4.0.8.tgz" + "mime-types": { + "version": "2.1.21", + "from": "mime-types@>=2.1.19 <2.2.0", + "resolved": "https://registry.npmjs.org/mime-types/-/mime-types-2.1.21.tgz" + }, + "oauth-sign": { + "version": "0.9.0", + "from": "oauth-sign@>=0.9.0 <0.10.0", + "resolved": "https://registry.npmjs.org/oauth-sign/-/oauth-sign-0.9.0.tgz" + }, + "request": { + "version": "2.88.0", + "from": "request@>=2.88.0 <3.0.0", + "resolved": "https://registry.npmjs.org/request/-/request-2.88.0.tgz" + }, + "safe-buffer": { + "version": "5.1.2", + "from": "safe-buffer@>=5.1.2 <6.0.0", + "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.1.2.tgz" + }, + "tough-cookie": { + "version": "2.4.3", + "from": "tough-cookie@>=2.4.3 <2.5.0", + "resolved": "https://registry.npmjs.org/tough-cookie/-/tough-cookie-2.4.3.tgz" + }, + "tunnel-agent": { + "version": "0.6.0", + "from": "tunnel-agent@>=0.6.0 <0.7.0", + "resolved": "https://registry.npmjs.org/tunnel-agent/-/tunnel-agent-0.6.0.tgz" + }, + "uuid": { + "version": "3.3.2", + "from": "uuid@>=3.3.2 <4.0.0", + "resolved": "https://registry.npmjs.org/uuid/-/uuid-3.3.2.tgz" } } }, - "lolex": { - "version": "3.0.0", - "from": "lolex@>=3.0.0 <4.0.0", - "resolved": "https://registry.npmjs.org/lolex/-/lolex-3.0.0.tgz" - }, "long": { "version": "4.0.0", "from": "long@>=4.0.0 <5.0.0", "resolved": "https://registry.npmjs.org/long/-/long-4.0.0.tgz" }, "lru-cache": { - "version": "2.7.3", - "from": "lru-cache@>=2.0.0 <3.0.0", - "resolved": "https://registry.npmjs.org/lru-cache/-/lru-cache-2.7.3.tgz" + "version": "5.1.1", + "from": "lru-cache@>=5.0.0 <6.0.0", + "resolved": "https://registry.npmjs.org/lru-cache/-/lru-cache-5.1.1.tgz" }, "lsmod": { "version": "1.0.0", @@ -1343,9 +1115,9 @@ "resolved": "https://registry.npmjs.org/methods/-/methods-1.1.2.tgz" }, "metrics-sharelatex": { - "version": "2.0.12", - "from": "git+https://github.com/sharelatex/metrics-sharelatex.git#v2.0.12", - "resolved": "git+https://github.com/sharelatex/metrics-sharelatex.git#3ac1621ef049e2f2d88a83b3a41011333d609662", + "version": "2.1.1", + "from": "metrics-sharelatex@2.1.1", + "resolved": "https://registry.npmjs.org/metrics-sharelatex/-/metrics-sharelatex-2.1.1.tgz", "dependencies": { "coffee-script": { "version": "1.6.0", @@ -1458,8 +1230,7 @@ "nan": { "version": "2.12.1", "from": "nan@>=2.0.8 <3.0.0", - "resolved": "https://registry.npmjs.org/nan/-/nan-2.12.1.tgz", - "optional": true + "resolved": "https://registry.npmjs.org/nan/-/nan-2.12.1.tgz" }, "native-or-bluebird": { "version": "1.1.2", @@ -1478,23 +1249,6 @@ "from": "negotiator@0.6.1", "resolved": "https://registry.npmjs.org/negotiator/-/negotiator-0.6.1.tgz" }, - "nise": { - "version": "1.4.8", - "from": "nise@>=1.4.7 <2.0.0", - "resolved": "https://registry.npmjs.org/nise/-/nise-1.4.8.tgz", - "dependencies": { - "lolex": { - "version": "2.7.5", - "from": "lolex@>=2.3.2 <3.0.0", - "resolved": "https://registry.npmjs.org/lolex/-/lolex-2.7.5.tgz" - }, - "path-to-regexp": { - "version": "1.7.0", - "from": "path-to-regexp@>=1.7.0 <2.0.0", - "resolved": "https://registry.npmjs.org/path-to-regexp/-/path-to-regexp-1.7.0.tgz" - } - } - }, "node-fetch": { "version": "2.3.0", "from": "node-fetch@>=2.2.0 <3.0.0", @@ -1505,11 +1259,6 @@ "from": "node-forge@>=0.7.4 <0.8.0", "resolved": "https://registry.npmjs.org/node-forge/-/node-forge-0.7.6.tgz" }, - "nopt": { - "version": "1.0.10", - "from": "nopt@>=1.0.10 <1.1.0", - "resolved": "https://registry.npmjs.org/nopt/-/nopt-1.0.10.tgz" - }, "oauth-sign": { "version": "0.3.0", "from": "oauth-sign@>=0.3.0 <0.4.0", @@ -1576,10 +1325,10 @@ "from": "path-to-regexp@0.1.7", "resolved": "https://registry.npmjs.org/path-to-regexp/-/path-to-regexp-0.1.7.tgz" }, - "pathval": { - "version": "1.1.0", - "from": "pathval@>=1.0.0 <2.0.0", - "resolved": "https://registry.npmjs.org/pathval/-/pathval-1.1.0.tgz" + "performance-now": { + "version": "2.1.0", + "from": "performance-now@>=2.1.0 <3.0.0", + "resolved": "https://registry.npmjs.org/performance-now/-/performance-now-2.1.0.tgz" }, "pify": { "version": "4.0.1", @@ -1616,11 +1365,15 @@ "from": "proxy-addr@>=2.0.3 <2.1.0", "resolved": "https://registry.npmjs.org/proxy-addr/-/proxy-addr-2.0.3.tgz" }, + "psl": { + "version": "1.1.31", + "from": "psl@>=1.1.24 <2.0.0", + "resolved": "https://registry.npmjs.org/psl/-/psl-1.1.31.tgz" + }, "punycode": { "version": "1.4.1", "from": "punycode@>=1.4.1 <2.0.0", - "resolved": "https://registry.npmjs.org/punycode/-/punycode-1.4.1.tgz", - "optional": true + "resolved": "https://registry.npmjs.org/punycode/-/punycode-1.4.1.tgz" }, "q": { "version": "0.9.2", @@ -1655,14 +1408,7 @@ "readable-stream": { "version": "2.3.6", "from": "readable-stream@>=2.0.0 <3.0.0", - "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-2.3.6.tgz", - "dependencies": { - "isarray": { - "version": "1.0.0", - "from": "isarray@>=1.0.0 <1.1.0", - "resolved": "https://registry.npmjs.org/isarray/-/isarray-1.0.0.tgz" - } - } + "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-2.3.6.tgz" }, "redis": { "version": "0.12.1", @@ -1698,8 +1444,8 @@ }, "redis-sharelatex": { "version": "1.0.5", - "from": "git+https://github.com/sharelatex/redis-sharelatex.git#v1.0.5", - "resolved": "git+https://github.com/sharelatex/redis-sharelatex.git#d6130c85bad845359729c45b8467d9d8a3ff44de", + "from": "https://registry.npmjs.org/redis-sharelatex/-/redis-sharelatex-1.0.5.tgz", + "resolved": "https://registry.npmjs.org/redis-sharelatex/-/redis-sharelatex-1.0.5.tgz", "dependencies": { "async": { "version": "2.6.1", @@ -1711,11 +1457,6 @@ "from": "coffee-script@1.8.0", "resolved": "https://registry.npmjs.org/coffee-script/-/coffee-script-1.8.0.tgz" }, - "lodash": { - "version": "4.17.11", - "from": "lodash@>=4.17.10 <5.0.0", - "resolved": "https://registry.npmjs.org/lodash/-/lodash-4.17.11.tgz" - }, "mkdirp": { "version": "0.3.5", "from": "mkdirp@>=0.3.5 <0.4.0", @@ -1753,12 +1494,13 @@ "require-like": { "version": "0.1.2", "from": "require-like@0.1.2", - "resolved": "https://registry.npmjs.org/require-like/-/require-like-0.1.2.tgz" + "resolved": "https://registry.npmjs.org/require-like/-/require-like-0.1.2.tgz", + "dev": true }, "resolve": { - "version": "1.9.0", + "version": "1.10.0", "from": "resolve@>=1.5.0 <2.0.0", - "resolved": "https://registry.npmjs.org/resolve/-/resolve-1.9.0.tgz" + "resolved": "https://registry.npmjs.org/resolve/-/resolve-1.10.0.tgz" }, "retry-axios": { "version": "0.3.2", @@ -1773,7 +1515,8 @@ "rimraf": { "version": "2.4.5", "from": "rimraf@~2.4.0", - "resolved": "https://registry.npmjs.org/rimraf/-/rimraf-2.4.5.tgz" + "resolved": "https://registry.npmjs.org/rimraf/-/rimraf-2.4.5.tgz", + "optional": true }, "safe-buffer": { "version": "5.1.1", @@ -1839,8 +1582,8 @@ }, "settings-sharelatex": { "version": "1.1.0", - "from": "git+https://github.com/sharelatex/settings-sharelatex.git#v1.1.0", - "resolved": "git+https://github.com/sharelatex/settings-sharelatex.git#93f63d029b52fef8825c3a401b2b6a7ba29b4750", + "from": "settings-sharelatex@1.1.0", + "resolved": "https://registry.npmjs.org/settings-sharelatex/-/settings-sharelatex-1.1.0.tgz", "dependencies": { "coffee-script": { "version": "1.6.0", @@ -1850,14 +1593,9 @@ } }, "shimmer": { - "version": "1.2.0", + "version": "1.2.1", "from": "shimmer@>=1.2.0 <2.0.0", - "resolved": "https://registry.npmjs.org/shimmer/-/shimmer-1.2.0.tgz" - }, - "sigmund": { - "version": "1.0.1", - "from": "sigmund@>=1.0.0 <1.1.0", - "resolved": "https://registry.npmjs.org/sigmund/-/sigmund-1.0.1.tgz" + "resolved": "https://registry.npmjs.org/shimmer/-/shimmer-1.2.1.tgz" }, "sinon": { "version": "1.5.2", @@ -1904,6 +1642,23 @@ "from": "split@>=1.0.0 <2.0.0", "resolved": "https://registry.npmjs.org/split/-/split-1.0.1.tgz" }, + "sshpk": { + "version": "1.16.1", + "from": "sshpk@>=1.7.0 <2.0.0", + "resolved": "https://registry.npmjs.org/sshpk/-/sshpk-1.16.1.tgz", + "dependencies": { + "asn1": { + "version": "0.2.4", + "from": "asn1@>=0.2.3 <0.3.0", + "resolved": "https://registry.npmjs.org/asn1/-/asn1-0.2.4.tgz" + }, + "assert-plus": { + "version": "1.0.0", + "from": "assert-plus@^1.0.0", + "resolved": "https://registry.npmjs.org/assert-plus/-/assert-plus-1.0.0.tgz" + } + } + }, "stack-trace": { "version": "0.0.9", "from": "stack-trace@0.0.9", @@ -1934,16 +1689,6 @@ "from": "string_decoder@>=1.1.1 <1.2.0", "resolved": "https://registry.npmjs.org/string_decoder/-/string_decoder-1.1.1.tgz" }, - "strip-ansi": { - "version": "0.3.0", - "from": "strip-ansi@>=0.3.0 <0.4.0", - "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-0.3.0.tgz" - }, - "supports-color": { - "version": "0.2.0", - "from": "supports-color@>=0.2.0 <0.3.0", - "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-0.2.0.tgz" - }, "symbol-observable": { "version": "1.2.0", "from": "symbol-observable@>=1.2.0 <2.0.0", @@ -1966,11 +1711,6 @@ } } }, - "text-encoding": { - "version": "0.6.4", - "from": "text-encoding@>=0.6.4 <0.7.0", - "resolved": "https://registry.npmjs.org/text-encoding/-/text-encoding-0.6.4.tgz" - }, "through": { "version": "2.3.8", "from": "through@>=2.0.0 <3.0.0", @@ -2004,6 +1744,11 @@ "resolved": "https://registry.npmjs.org/tunnel-agent/-/tunnel-agent-0.3.0.tgz", "optional": true }, + "tweetnacl": { + "version": "0.14.5", + "from": "tweetnacl@>=0.14.0 <0.15.0", + "resolved": "https://registry.npmjs.org/tweetnacl/-/tweetnacl-0.14.5.tgz" + }, "type-detect": { "version": "0.1.1", "from": "type-detect@0.1.1", @@ -2031,16 +1776,23 @@ "from": "underscore@>=1.7.0 <1.8.0", "resolved": "https://registry.npmjs.org/underscore/-/underscore-1.7.0.tgz" }, - "underscore.string": { - "version": "2.2.1", - "from": "underscore.string@>=2.2.1 <2.3.0", - "resolved": "https://registry.npmjs.org/underscore.string/-/underscore.string-2.2.1.tgz" - }, "unpipe": { "version": "1.0.0", "from": "unpipe@1.0.0", "resolved": "https://registry.npmjs.org/unpipe/-/unpipe-1.0.0.tgz" }, + "uri-js": { + "version": "4.2.2", + "from": "uri-js@>=4.2.2 <5.0.0", + "resolved": "https://registry.npmjs.org/uri-js/-/uri-js-4.2.2.tgz", + "dependencies": { + "punycode": { + "version": "2.1.1", + "from": "punycode@>=2.1.0 <3.0.0", + "resolved": "https://registry.npmjs.org/punycode/-/punycode-2.1.1.tgz" + } + } + }, "util-deprecate": { "version": "1.0.2", "from": "util-deprecate@>=1.0.1 <1.1.0", @@ -2061,10 +1813,17 @@ "from": "vary@>=1.1.2 <1.2.0", "resolved": "https://registry.npmjs.org/vary/-/vary-1.1.2.tgz" }, - "which": { - "version": "1.0.9", - "from": "which@>=1.0.5 <1.1.0", - "resolved": "https://registry.npmjs.org/which/-/which-1.0.9.tgz" + "verror": { + "version": "1.10.0", + "from": "verror@1.10.0", + "resolved": "https://registry.npmjs.org/verror/-/verror-1.10.0.tgz", + "dependencies": { + "assert-plus": { + "version": "1.0.0", + "from": "assert-plus@^1.0.0", + "resolved": "https://registry.npmjs.org/assert-plus/-/assert-plus-1.0.0.tgz" + } + } }, "wrappy": { "version": "1.0.2", diff --git a/services/real-time/package.json b/services/real-time/package.json index 6268437247..ea519b24fc 100644 --- a/services/real-time/package.json +++ b/services/real-time/package.json @@ -28,12 +28,12 @@ "cookie-parser": "^1.3.3", "express": "^4.10.1", "express-session": "^1.9.1", - "logger-sharelatex": "git+https://github.com/sharelatex/logger-sharelatex.git#v1.5.9", - "metrics-sharelatex": "git+https://github.com/sharelatex/metrics-sharelatex.git#v2.0.12", - "redis-sharelatex": "git+https://github.com/sharelatex/redis-sharelatex.git#v1.0.5", + "logger-sharelatex": "^1.6.0", + "metrics-sharelatex": "^2.1.1", + "redis-sharelatex": "https://registry.npmjs.org/redis-sharelatex/-/redis-sharelatex-1.0.5.tgz", "request": "~2.34.0", "session.socket.io": "^0.1.6", - "settings-sharelatex": "git+https://github.com/sharelatex/settings-sharelatex.git#v1.1.0", + "settings-sharelatex": "^1.1.0", "socket.io": "0.9.16", "socket.io-client": "^0.9.16" }, From 0939a558d7b178c6c496dfece1a410d54195b859 Mon Sep 17 00:00:00 2001 From: Henry Oswald Date: Fri, 1 Feb 2019 19:27:56 +0000 Subject: [PATCH 174/491] point real time to web api --- services/real-time/config/settings.defaults.coffee | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/services/real-time/config/settings.defaults.coffee b/services/real-time/config/settings.defaults.coffee index ed9fa4f90b..6d5e00e658 100644 --- a/services/real-time/config/settings.defaults.coffee +++ b/services/real-time/config/settings.defaults.coffee @@ -29,7 +29,7 @@ settings = apis: web: - url: "http://#{process.env['WEB_HOST'] or "localhost"}:#{process.env['WEB_PORT'] or 3000}" + url: "http://#{process.env['WEB_API_HOST'] or process.env['WEB_HOST'] or "localhost"}:#{process.env['WEB_API_PORT'] or process.env['WEB_PORT'] or 3000}" user: process.env['WEB_API_USER'] or "sharelatex" pass: process.env['WEB_API_PASSWORD'] or "password" documentupdater: From a07e516f651c6144d64d9380bf41d1bb92e640a7 Mon Sep 17 00:00:00 2001 From: Henry Oswald Date: Mon, 4 Feb 2019 09:47:11 +0000 Subject: [PATCH 175/491] update redis key --- services/real-time/config/settings.defaults.coffee | 6 +++--- .../test/acceptance/coffee/ApplyUpdateTests.coffee | 12 +++++++----- 2 files changed, 10 insertions(+), 8 deletions(-) diff --git a/services/real-time/config/settings.defaults.coffee b/services/real-time/config/settings.defaults.coffee index 6d5e00e658..93a980931e 100644 --- a/services/real-time/config/settings.defaults.coffee +++ b/services/real-time/config/settings.defaults.coffee @@ -5,15 +5,15 @@ settings = port: process.env['REAL_TIME_REDIS_PORT'] or process.env['REDIS_PORT'] or "6379" password: process.env["REAL_TIME_REDIS_PASSWORD"] or process.env["REDIS_PASSWORD"] or "" key_schema: - clientsInProject: ({project_id}) -> "clients_in_project:#{project_id}" - connectedUser: ({project_id, client_id})-> "connected_user:#{project_id}:#{client_id}" + clientsInProject: ({project_id}) -> "clients_in_project:{#{project_id}}" + connectedUser: ({project_id, client_id})-> "connected_user:{#{project_id}}:#{client_id}" documentupdater: host: process.env['DOC_UPDATER_REDIS_HOST'] or process.env['REDIS_HOST'] or "localhost" port: process.env['DOC_UPDATER_REDIS_PORT'] or process.env['REDIS_PORT'] or "6379" password: process.env["DOC_UPDATER_REDIS_PASSWORD"] or process.env["REDIS_PASSWORD"] or "" key_schema: - pendingUpdates: ({doc_id}) -> "PendingUpdates:#{doc_id}" + pendingUpdates: ({doc_id}) -> "PendingUpdates:{#{doc_id}}" websessions: host: process.env['WEB_REDIS_HOST'] or process.env['REDIS_HOST'] or "localhost" diff --git a/services/real-time/test/acceptance/coffee/ApplyUpdateTests.coffee b/services/real-time/test/acceptance/coffee/ApplyUpdateTests.coffee index 2509f21108..d9addc990d 100644 --- a/services/real-time/test/acceptance/coffee/ApplyUpdateTests.coffee +++ b/services/real-time/test/acceptance/coffee/ApplyUpdateTests.coffee @@ -10,6 +10,8 @@ settings = require "settings-sharelatex" redis = require "redis-sharelatex" rclient = redis.createClient(settings.redis.websessions) +redisSettings = settings.redis + describe "applyOtUpdate", -> before -> @update = { @@ -48,7 +50,7 @@ describe "applyOtUpdate", -> done() it "should push the update into redis", (done) -> - rclient.lrange "PendingUpdates:#{@doc_id}", 0, -1, (error, [update]) => + rclient.lrange redisSettings.documentupdater.key_schema.pendingUpdates({@doc_id}), 0, -1, (error, [update]) => update = JSON.parse(update) update.op.should.deep.equal @update.op update.meta.should.deep.equal { @@ -61,7 +63,7 @@ describe "applyOtUpdate", -> async.series [ (cb) => rclient.del "pending-updates-list", cb (cb) => rclient.del "DocsWithPendingUpdates", "#{@project_id}:#{@doc_id}", cb - (cb) => rclient.del "PendingUpdates:#{@doc_id}", cb + (cb) => rclient.del redisSettings.documentupdater.key_schema.pendingUpdates(@doc_id), cb ], done describe "when authorized to read-only with an edit update", -> @@ -102,7 +104,7 @@ describe "applyOtUpdate", -> , 300 it "should not put the update in redis", (done) -> - rclient.llen "PendingUpdates:#{@doc_id}", (error, len) => + rclient.llen redisSettings.documentupdater.key_schema.pendingUpdates({@doc_id}), (error, len) => len.should.equal 0 done() @@ -142,7 +144,7 @@ describe "applyOtUpdate", -> done() it "should push the update into redis", (done) -> - rclient.lrange "PendingUpdates:#{@doc_id}", 0, -1, (error, [update]) => + rclient.lrange redisSettings.documentupdater.key_schema.pendingUpdates({@doc_id}), 0, -1, (error, [update]) => update = JSON.parse(update) update.op.should.deep.equal @comment_update.op update.meta.should.deep.equal { @@ -155,5 +157,5 @@ describe "applyOtUpdate", -> async.series [ (cb) => rclient.del "pending-updates-list", cb (cb) => rclient.del "DocsWithPendingUpdates", "#{@project_id}:#{@doc_id}", cb - (cb) => rclient.del "PendingUpdates:#{@doc_id}", cb + (cb) => rclient.del redisSettings.documentupdater.key_schema.pendingUpdates({@doc_id}), cb ], done \ No newline at end of file From 37d1c9605119ce73fd6dbc2bf3a3710675ae5c50 Mon Sep 17 00:00:00 2001 From: Henry Oswald Date: Mon, 4 Feb 2019 16:59:36 +0000 Subject: [PATCH 176/491] add logging --- services/real-time/app/coffee/DocumentUpdaterController.coffee | 1 + 1 file changed, 1 insertion(+) diff --git a/services/real-time/app/coffee/DocumentUpdaterController.coffee b/services/real-time/app/coffee/DocumentUpdaterController.coffee index 451dc812bc..72078d71d2 100644 --- a/services/real-time/app/coffee/DocumentUpdaterController.coffee +++ b/services/real-time/app/coffee/DocumentUpdaterController.coffee @@ -26,6 +26,7 @@ module.exports = DocumentUpdaterController = DocumentUpdaterController._processErrorFromDocumentUpdater(io, message.doc_id, message.error, message) _applyUpdateFromDocumentUpdater: (io, doc_id, update) -> + logger.log({doc_id}, "apply update from doc updater") for client in io.sockets.clients(doc_id) if client.id == update.meta.source logger.log doc_id: doc_id, version: update.v, source: update.meta?.source, "distributing update to sender" From d85bf5cedbec2279407da9184b8e279a8cf59002 Mon Sep 17 00:00:00 2001 From: Henry Oswald Date: Wed, 6 Feb 2019 15:26:12 +0000 Subject: [PATCH 177/491] remove extra logging line --- services/real-time/app/coffee/DocumentUpdaterController.coffee | 1 - 1 file changed, 1 deletion(-) diff --git a/services/real-time/app/coffee/DocumentUpdaterController.coffee b/services/real-time/app/coffee/DocumentUpdaterController.coffee index 72078d71d2..451dc812bc 100644 --- a/services/real-time/app/coffee/DocumentUpdaterController.coffee +++ b/services/real-time/app/coffee/DocumentUpdaterController.coffee @@ -26,7 +26,6 @@ module.exports = DocumentUpdaterController = DocumentUpdaterController._processErrorFromDocumentUpdater(io, message.doc_id, message.error, message) _applyUpdateFromDocumentUpdater: (io, doc_id, update) -> - logger.log({doc_id}, "apply update from doc updater") for client in io.sockets.clients(doc_id) if client.id == update.meta.source logger.log doc_id: doc_id, version: update.v, source: update.meta?.source, "distributing update to sender" From 1fc1b4206e5f927145d653f7fa835b8230a825db Mon Sep 17 00:00:00 2001 From: Henry Oswald Date: Thu, 7 Feb 2019 13:57:38 +0000 Subject: [PATCH 178/491] add shutDownInProgress check into sig listening --- services/real-time/app.coffee | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/services/real-time/app.coffee b/services/real-time/app.coffee index 1035f83b68..ab0ac3627a 100644 --- a/services/real-time/app.coffee +++ b/services/real-time/app.coffee @@ -103,11 +103,17 @@ forceDrain = -> DrainManager.startDrain(io, 4) , Settings.forceDrainMsDelay +shutDownInProgress = false if Settings.forceDrainMsDelay? Settings.forceDrainMsDelay = parseInt(Settings.forceDrainMsDelay, 10) logger.log forceDrainMsDelay: Settings.forceDrainMsDelay,"forceDrainMsDelay enabled" for signal in ['SIGINT', 'SIGHUP', 'SIGQUIT', 'SIGUSR1', 'SIGUSR2', 'SIGTERM', 'SIGABRT'] process.on signal, -> - logger.log signal: signal, "received interrupt, cleaning up" - shutdownCleanly(signal) - forceDrain() + if shutDownInProgress + logger.log signal: signal, "shutdown already in progress, ignoring signal" + return + else + shutDownInProgress = true + logger.log signal: signal, "received interrupt, cleaning up" + shutdownCleanly(signal) + forceDrain() From cb12e1c6f6574b0c1f6558fef7385e7ac7f6dfe0 Mon Sep 17 00:00:00 2001 From: Chrystal Griffiths Date: Fri, 8 Feb 2019 15:39:51 +0000 Subject: [PATCH 179/491] Send an empty string for every nameless user --- .../app/coffee/WebsocketController.coffee | 36 +++++++++---------- .../coffee/WebsocketControllerTests.coffee | 34 ++++++++++++++---- 2 files changed, 44 insertions(+), 26 deletions(-) diff --git a/services/real-time/app/coffee/WebsocketController.coffee b/services/real-time/app/coffee/WebsocketController.coffee index b51e86c495..734dfc3005 100644 --- a/services/real-time/app/coffee/WebsocketController.coffee +++ b/services/real-time/app/coffee/WebsocketController.coffee @@ -145,26 +145,24 @@ module.exports = WebsocketController = cursorData.id = client.id cursorData.user_id = user_id if user_id? cursorData.email = email if email? - if first_name or last_name - cursorData.name = if first_name && last_name - "#{first_name} #{last_name}" - else if first_name - first_name - else if last_name - last_name - ConnectedUsersManager.updateUserPosition(project_id, client.id, { - first_name: first_name, - last_name: last_name, - email: email, - _id: user_id - }, { - row: cursorData.row, - column: cursorData.column, - doc_id: cursorData.doc_id - }, callback) + cursorData.name = if first_name && last_name + "#{first_name} #{last_name}" + else if first_name + first_name + else if last_name + last_name else - cursorData.name = "Anonymous" - callback() + "" + ConnectedUsersManager.updateUserPosition(project_id, client.id, { + first_name: first_name, + last_name: last_name, + email: email, + _id: user_id + }, { + row: cursorData.row, + column: cursorData.column, + doc_id: cursorData.doc_id + }, callback) WebsocketLoadBalancer.emitToRoom(project_id, "clientTracking.clientUpdated", cursorData) getConnectedUsers: (client, callback = (error, users) ->) -> diff --git a/services/real-time/test/unit/coffee/WebsocketControllerTests.coffee b/services/real-time/test/unit/coffee/WebsocketControllerTests.coffee index b6a4069c67..0a0bd50843 100644 --- a/services/real-time/test/unit/coffee/WebsocketControllerTests.coffee +++ b/services/real-time/test/unit/coffee/WebsocketControllerTests.coffee @@ -426,7 +426,7 @@ describe 'WebsocketController', -> doc_id: @doc_id }).should.equal true done() - + it "should increment the update-client-position metric at 0.1 frequency", -> @metrics.inc.calledWith("editor.update-client-position", 0.1).should.equal true @@ -509,6 +509,30 @@ describe 'WebsocketController', -> it "should increment the update-client-position metric at 0.1 frequency", -> @metrics.inc.calledWith("editor.update-client-position", 0.1).should.equal true + describe "with a logged in user who has no names set", -> + beforeEach -> + @clientParams = { + project_id: @project_id + first_name: undefined + last_name: undefined + email: @email = "joe@example.com" + user_id: @user_id = "user-id-123" + } + @client.get = (param, callback) => callback null, @clientParams[param] + @WebsocketController.updateClientPosition @client, @update + + it "should send the update to the project name with no name", -> + @WebsocketLoadBalancer.emitToRoom + .calledWith(@project_id, "clientTracking.clientUpdated", { + doc_id: @doc_id, + id: @client.id, + user_id: @user_id, + name: "", + row: @row, + column: @column, + email: @email + }) + .should.equal true describe "with an anonymous user", -> @@ -519,20 +543,16 @@ describe 'WebsocketController', -> @client.get = (param, callback) => callback null, @clientParams[param] @WebsocketController.updateClientPosition @client, @update - it "should send the update to the project room with an anonymous name", -> + it "should send the update to the project room with no name", -> @WebsocketLoadBalancer.emitToRoom .calledWith(@project_id, "clientTracking.clientUpdated", { doc_id: @doc_id, id: @client.id - name: "Anonymous" + name: "" row: @row column: @column }) .should.equal true - - it "should not send cursor data to the connected user manager", (done)-> - @ConnectedUsersManager.updateUserPosition.called.should.equal false - done() describe "applyOtUpdate", -> beforeEach -> From 2ec760403f18a78cd13f3a2b8ead1d718ec03ed2 Mon Sep 17 00:00:00 2001 From: Chrystal Griffiths Date: Mon, 11 Feb 2019 11:52:14 +0000 Subject: [PATCH 180/491] Revert to method not sending cursorData because of duplication --- .../app/coffee/WebsocketController.coffee | 39 ++++++++++--------- 1 file changed, 20 insertions(+), 19 deletions(-) diff --git a/services/real-time/app/coffee/WebsocketController.coffee b/services/real-time/app/coffee/WebsocketController.coffee index 734dfc3005..e5c4f0c1c0 100644 --- a/services/real-time/app/coffee/WebsocketController.coffee +++ b/services/real-time/app/coffee/WebsocketController.coffee @@ -129,7 +129,6 @@ module.exports = WebsocketController = # after the initial joinDoc since we know they are already authorised. ## AuthorizationManager.removeAccessToDoc client, doc_id callback() - updateClientPosition: (client, cursorData, callback = (error) ->) -> metrics.inc "editor.update-client-position", 0.1 Utils.getClientAttributes client, [ @@ -145,24 +144,26 @@ module.exports = WebsocketController = cursorData.id = client.id cursorData.user_id = user_id if user_id? cursorData.email = email if email? - cursorData.name = if first_name && last_name - "#{first_name} #{last_name}" - else if first_name - first_name - else if last_name - last_name + if first_name or last_name + cursorData.name = if first_name && last_name + "#{first_name} #{last_name}" + else if first_name + first_name + else if last_name + last_name + ConnectedUsersManager.updateUserPosition(project_id, client.id, { + first_name: first_name, + last_name: last_name, + email: email, + _id: user_id + }, { + row: cursorData.row, + column: cursorData.column, + doc_id: cursorData.doc_id + }, callback) else - "" - ConnectedUsersManager.updateUserPosition(project_id, client.id, { - first_name: first_name, - last_name: last_name, - email: email, - _id: user_id - }, { - row: cursorData.row, - column: cursorData.column, - doc_id: cursorData.doc_id - }, callback) + cursorData.name = "" + callback() WebsocketLoadBalancer.emitToRoom(project_id, "clientTracking.clientUpdated", cursorData) getConnectedUsers: (client, callback = (error, users) ->) -> @@ -221,4 +222,4 @@ module.exports = WebsocketController = for op in update.op if !op.c? return false - return true + return true \ No newline at end of file From bb06f82e0433f8ca369b65191e56c1f44fcc1985 Mon Sep 17 00:00:00 2001 From: Chrystal Griffiths Date: Tue, 12 Feb 2019 14:00:47 +0000 Subject: [PATCH 181/491] Still send cursorData for logged in users --- .../app/coffee/WebsocketController.coffee | 19 +++++++++++-------- .../coffee/WebsocketControllerTests.coffee | 4 ++++ 2 files changed, 15 insertions(+), 8 deletions(-) diff --git a/services/real-time/app/coffee/WebsocketController.coffee b/services/real-time/app/coffee/WebsocketController.coffee index e5c4f0c1c0..b4a682f294 100644 --- a/services/real-time/app/coffee/WebsocketController.coffee +++ b/services/real-time/app/coffee/WebsocketController.coffee @@ -52,6 +52,10 @@ module.exports = WebsocketController = metrics.inc "editor.leave-project" Utils.getClientAttributes client, ["project_id", "user_id"], (error, {project_id, user_id}) -> return callback(error) if error? + + logger.log {project_id, user_id, client_id: client.id}, "client leaving project" + WebsocketLoadBalancer.emitToRoom project_id, "clientTracking.clientDisconnected", client.id + # bail out if the client had not managed to authenticate or join # the project. Prevents downstream errors in docupdater from # flushProjectToMongoAndDelete with null project_id. @@ -61,9 +65,6 @@ module.exports = WebsocketController = if not project_id? logger.log {user_id: user_id, client_id: client.id}, "client leaving, not in project" return callback() - - logger.log {project_id, user_id, client_id: client.id}, "client leaving project" - WebsocketLoadBalancer.emitToRoom project_id, "clientTracking.clientDisconnected", client.id # We can do this in the background ConnectedUsersManager.markUserAsDisconnected project_id, client.id, (err) -> @@ -144,13 +145,18 @@ module.exports = WebsocketController = cursorData.id = client.id cursorData.user_id = user_id if user_id? cursorData.email = email if email? - if first_name or last_name + if !user_id or user_id == 'anonymous-user' + cursorData.name = "" + callback() + else cursorData.name = if first_name && last_name "#{first_name} #{last_name}" else if first_name first_name else if last_name last_name + else + "" ConnectedUsersManager.updateUserPosition(project_id, client.id, { first_name: first_name, last_name: last_name, @@ -160,10 +166,7 @@ module.exports = WebsocketController = row: cursorData.row, column: cursorData.column, doc_id: cursorData.doc_id - }, callback) - else - cursorData.name = "" - callback() + }, callback) WebsocketLoadBalancer.emitToRoom(project_id, "clientTracking.clientUpdated", cursorData) getConnectedUsers: (client, callback = (error, users) ->) -> diff --git a/services/real-time/test/unit/coffee/WebsocketControllerTests.coffee b/services/real-time/test/unit/coffee/WebsocketControllerTests.coffee index 0a0bd50843..5b36057813 100644 --- a/services/real-time/test/unit/coffee/WebsocketControllerTests.coffee +++ b/services/real-time/test/unit/coffee/WebsocketControllerTests.coffee @@ -554,6 +554,10 @@ describe 'WebsocketController', -> }) .should.equal true + it "should not send cursor data to the connected user manager", (done)-> + @ConnectedUsersManager.updateUserPosition.called.should.equal false + done() + describe "applyOtUpdate", -> beforeEach -> @update = {op: {p: 12, t: "foo"}} From 26acdfd072ead1549d925288e0059630b5c33465 Mon Sep 17 00:00:00 2001 From: Chrystal Griffiths Date: Tue, 12 Feb 2019 14:06:59 +0000 Subject: [PATCH 182/491] Add comment explaining why not sending anon data up --- services/real-time/app/coffee/WebsocketController.coffee | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/services/real-time/app/coffee/WebsocketController.coffee b/services/real-time/app/coffee/WebsocketController.coffee index b4a682f294..008a5560a7 100644 --- a/services/real-time/app/coffee/WebsocketController.coffee +++ b/services/real-time/app/coffee/WebsocketController.coffee @@ -137,7 +137,7 @@ module.exports = WebsocketController = ], (error, {project_id, first_name, last_name, email, user_id}) -> return callback(error) if error? logger.log {user_id, project_id, client_id: client.id, cursorData: cursorData}, "updating client position" - + AuthorizationManager.assertClientCanViewProjectAndDoc client, cursorData.doc_id, (error) -> if error? logger.warn {err: error, client_id: client.id, project_id, user_id}, "silently ignoring unauthorized updateClientPosition. Client likely hasn't called joinProject yet." @@ -145,10 +145,11 @@ module.exports = WebsocketController = cursorData.id = client.id cursorData.user_id = user_id if user_id? cursorData.email = email if email? + # Don't store anonymous users in redis to avoid influx if !user_id or user_id == 'anonymous-user' cursorData.name = "" callback() - else + else cursorData.name = if first_name && last_name "#{first_name} #{last_name}" else if first_name @@ -166,7 +167,7 @@ module.exports = WebsocketController = row: cursorData.row, column: cursorData.column, doc_id: cursorData.doc_id - }, callback) + }, callback) WebsocketLoadBalancer.emitToRoom(project_id, "clientTracking.clientUpdated", cursorData) getConnectedUsers: (client, callback = (error, users) ->) -> From 6fb6086ba10766343639330ecc2265b23ee562c3 Mon Sep 17 00:00:00 2001 From: Henry Oswald Date: Tue, 12 Feb 2019 14:28:42 +0000 Subject: [PATCH 183/491] remove console.log --- services/real-time/app.coffee | 2 -- 1 file changed, 2 deletions(-) diff --git a/services/real-time/app.coffee b/services/real-time/app.coffee index ab0ac3627a..2661ce8430 100644 --- a/services/real-time/app.coffee +++ b/services/real-time/app.coffee @@ -2,8 +2,6 @@ Metrics = require("metrics-sharelatex") Settings = require "settings-sharelatex" Metrics.initialize(Settings.appName or "real-time") -console.log Settings.redis - logger = require "logger-sharelatex" logger.initialize("real-time") Metrics.event_loop.monitor(logger) From 507c4e5ce2a3b463c395915ab63f19d4b0d2f96a Mon Sep 17 00:00:00 2001 From: Chrystal Griffiths Date: Fri, 15 Feb 2019 15:18:17 +0000 Subject: [PATCH 184/491] Fix test to accommodate changes to anonymous users --- .../real-time/test/acceptance/coffee/ClientTrackingTests.coffee | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/services/real-time/test/acceptance/coffee/ClientTrackingTests.coffee b/services/real-time/test/acceptance/coffee/ClientTrackingTests.coffee index dc00035382..d13663ed70 100644 --- a/services/real-time/test/acceptance/coffee/ClientTrackingTests.coffee +++ b/services/real-time/test/acceptance/coffee/ClientTrackingTests.coffee @@ -141,6 +141,6 @@ describe "clientTracking", -> doc_id: @doc_id id: @anonymous.socket.sessionid user_id: "anonymous-user" - name: "Anonymous" + name: "" } ] From 26e903f384e8fdfcea921579a80da405ac051a91 Mon Sep 17 00:00:00 2001 From: Henry Oswald Date: Fri, 15 Feb 2019 15:23:59 +0000 Subject: [PATCH 185/491] setup continualPubsubTraffic this keeps the pub sub channel ticking along happily --- services/real-time/app.coffee | 25 +++++++++++++++++++ .../coffee/DocumentUpdaterController.coffee | 2 ++ .../app/coffee/WebsocketLoadBalancer.coffee | 2 ++ .../real-time/config/settings.defaults.coffee | 2 ++ 4 files changed, 31 insertions(+) diff --git a/services/real-time/app.coffee b/services/real-time/app.coffee index 2661ce8430..89f0cdfb36 100644 --- a/services/real-time/app.coffee +++ b/services/real-time/app.coffee @@ -1,6 +1,7 @@ Metrics = require("metrics-sharelatex") Settings = require "settings-sharelatex" Metrics.initialize(Settings.appName or "real-time") +async = require("async") logger = require "logger-sharelatex" logger.initialize("real-time") @@ -115,3 +116,27 @@ if Settings.forceDrainMsDelay? logger.log signal: signal, "received interrupt, cleaning up" shutdownCleanly(signal) forceDrain() + + + +if Settings.continualPubsubTraffic + console.log "continualPubsubTraffic enabled" + + pubSubClient = redis.createClient(Settings.redis.documentupdater) + + publishJob = (channel, cb)-> + json = JSON.stringify({health_check:true, date: new Date().toString()}) + logger.debug {channel:channel}, "sending pub to keep connection alive" + pubSubClient.publish channel, json, (err)-> + if err? + logger.err {err, channel}, "error publishing pubsub traffic to redis" + cb(err) + + runPubSubTraffic = -> + async.map ["applied-ops", "editor-events"], publishJob, (err)-> + setTimeout(runPubSubTraffic, 1000 * 60) + + runPubSubTraffic() + + + diff --git a/services/real-time/app/coffee/DocumentUpdaterController.coffee b/services/real-time/app/coffee/DocumentUpdaterController.coffee index 451dc812bc..8396be263a 100644 --- a/services/real-time/app/coffee/DocumentUpdaterController.coffee +++ b/services/real-time/app/coffee/DocumentUpdaterController.coffee @@ -24,6 +24,8 @@ module.exports = DocumentUpdaterController = DocumentUpdaterController._applyUpdateFromDocumentUpdater(io, message.doc_id, message.op) else if message.error? DocumentUpdaterController._processErrorFromDocumentUpdater(io, message.doc_id, message.error, message) + else if message.health_check? + logger.debug {message}, "got health check message in applied ops channel" _applyUpdateFromDocumentUpdater: (io, doc_id, update) -> for client in io.sockets.clients(doc_id) diff --git a/services/real-time/app/coffee/WebsocketLoadBalancer.coffee b/services/real-time/app/coffee/WebsocketLoadBalancer.coffee index 56ce8e5d30..c30a3d3e85 100644 --- a/services/real-time/app/coffee/WebsocketLoadBalancer.coffee +++ b/services/real-time/app/coffee/WebsocketLoadBalancer.coffee @@ -37,4 +37,6 @@ module.exports = WebsocketLoadBalancer = io.sockets.emit(message.message, message.payload...) else if message.room_id? io.sockets.in(message.room_id).emit(message.message, message.payload...) + else if message.health_check? + logger.debug {message}, "got health check message in editor events channel" diff --git a/services/real-time/config/settings.defaults.coffee b/services/real-time/config/settings.defaults.coffee index 93a980931e..0fe6fa7be1 100644 --- a/services/real-time/config/settings.defaults.coffee +++ b/services/real-time/config/settings.defaults.coffee @@ -44,6 +44,8 @@ settings = forceDrainMsDelay: process.env['FORCE_DRAIN_MS_DELAY'] or false + continualPubsubTraffic: process.env['CONTINUAL_PUBSUB_TRAFFIC'] or false + # console.log settings.redis module.exports = settings \ No newline at end of file From b9e3853a47645529e292083b41d847b5b2e4def0 Mon Sep 17 00:00:00 2001 From: Henry Oswald Date: Tue, 5 Mar 2019 17:33:52 +0000 Subject: [PATCH 186/491] add sentry into settings.defaults --- services/real-time/config/settings.defaults.coffee | 2 ++ 1 file changed, 2 insertions(+) diff --git a/services/real-time/config/settings.defaults.coffee b/services/real-time/config/settings.defaults.coffee index 0fe6fa7be1..28c51f79be 100644 --- a/services/real-time/config/settings.defaults.coffee +++ b/services/real-time/config/settings.defaults.coffee @@ -46,6 +46,8 @@ settings = continualPubsubTraffic: process.env['CONTINUAL_PUBSUB_TRAFFIC'] or false + sentry: + dsn: process.env.SENTRY_DSN # console.log settings.redis module.exports = settings \ No newline at end of file From befe4be5171392e7a92c87cece2501e68890228b Mon Sep 17 00:00:00 2001 From: Brian Gough Date: Tue, 19 Mar 2019 10:55:12 +0000 Subject: [PATCH 187/491] add check for duplicate events --- .../real-time/app/coffee/EventLogger.coffee | 46 ++++++++++++++++ .../app/coffee/WebsocketLoadBalancer.coffee | 3 ++ .../test/unit/coffee/EventLoggerTests.coffee | 52 +++++++++++++++++++ .../coffee/WebsocketLoadBalancerTests.coffee | 1 + 4 files changed, 102 insertions(+) create mode 100644 services/real-time/app/coffee/EventLogger.coffee create mode 100644 services/real-time/test/unit/coffee/EventLoggerTests.coffee diff --git a/services/real-time/app/coffee/EventLogger.coffee b/services/real-time/app/coffee/EventLogger.coffee new file mode 100644 index 0000000000..773c9dfa9b --- /dev/null +++ b/services/real-time/app/coffee/EventLogger.coffee @@ -0,0 +1,46 @@ +logger = require 'logger-sharelatex' + +# keep track of message counters to detect duplicate and out of order events +# messsage ids have the format "UNIQUEHOSTKEY-COUNTER" + +EVENT_LOG_COUNTER = {} +EVENT_LOG_TIMESTAMP = {} +EVENT_COUNT = 0 + +module.exports = EventLogger = + + MAX_EVENTS_BEFORE_CLEAN: 100000 + MAX_STALE_TIME_IN_MS: 3600 * 1000 + + checkEventOrder: (message_id, message) -> + return if typeof(message_id) isnt 'string' + [key, count] = message_id.split("-", 2) + count = parseInt(count, 10) + if !count # ignore checks if counter is not present + return + # store the last count in a hash for each host + previous = EventLogger._storeEventCount(key, count) + if !previous? || count == (previous + 1) + return # order is ok + if (count == previous) + logger.error {key:key, previous: previous, count:count, message:message}, "duplicate event" + return "duplicate" + else + logger.error {key:key, previous: previous, count:count, message:message}, "events out of order" + return # out of order + + _storeEventCount: (key, count) -> + previous = EVENT_LOG_COUNTER[key] + now = Date.now() + EVENT_LOG_COUNTER[key] = count + EVENT_LOG_TIMESTAMP[key] = now + # periodically remove old counts + if (++EVENT_COUNT % EventLogger.MAX_EVENTS_BEFORE_CLEAN) == 0 + EventLogger._cleanEventStream(now) + return previous + + _cleanEventStream: (now) -> + for key, timestamp of EVENT_LOG_TIMESTAMP + if (now - timestamp) > EventLogger.MAX_STALE_TIME_IN_MS + delete EVENT_LOG_COUNTER[key] + delete EVENT_LOG_TIMESTAMP[key] \ No newline at end of file diff --git a/services/real-time/app/coffee/WebsocketLoadBalancer.coffee b/services/real-time/app/coffee/WebsocketLoadBalancer.coffee index c30a3d3e85..13137805fd 100644 --- a/services/real-time/app/coffee/WebsocketLoadBalancer.coffee +++ b/services/real-time/app/coffee/WebsocketLoadBalancer.coffee @@ -4,6 +4,7 @@ redis = require("redis-sharelatex") SafeJsonParse = require "./SafeJsonParse" rclientPub = redis.createClient(Settings.redis.realtime) rclientSub = redis.createClient(Settings.redis.realtime) +EventLogger = require "./EventLogger" module.exports = WebsocketLoadBalancer = rclientPub: rclientPub @@ -36,6 +37,8 @@ module.exports = WebsocketLoadBalancer = if message.room_id == "all" io.sockets.emit(message.message, message.payload...) else if message.room_id? + if message._id? + EventLogger.checkEventOrder(message._id, message) io.sockets.in(message.room_id).emit(message.message, message.payload...) else if message.health_check? logger.debug {message}, "got health check message in editor events channel" diff --git a/services/real-time/test/unit/coffee/EventLoggerTests.coffee b/services/real-time/test/unit/coffee/EventLoggerTests.coffee new file mode 100644 index 0000000000..ce955a8e7d --- /dev/null +++ b/services/real-time/test/unit/coffee/EventLoggerTests.coffee @@ -0,0 +1,52 @@ +require('chai').should() +expect = require("chai").expect +SandboxedModule = require('sandboxed-module') +modulePath = '../../../app/js/EventLogger' +sinon = require("sinon") +tk = require "timekeeper" + +describe 'EventLogger', -> + beforeEach -> + @start = Date.now() + tk.freeze(new Date(@start)) + @EventLogger = SandboxedModule.require modulePath, requires: + "logger-sharelatex": @logger = {error: sinon.stub()} + @id_1 = "abc-1" + @message_1 = "message-1" + @id_2 = "abc-2" + @message_2 = "message-2" + + afterEach -> + tk.reset() + + describe 'checkEventOrder', -> + + it 'should accept events in order', -> + @EventLogger.checkEventOrder(@id_1, @message_1) + status = @EventLogger.checkEventOrder(@id_2, @message_2) + expect(status).to.be.undefined + + it 'should return "duplicate" for the same event', -> + @EventLogger.checkEventOrder(@id_1, @message_1) + status = @EventLogger.checkEventOrder(@id_1, @message_1) + expect(status).to.equal "duplicate" + + it 'should log an error for out of order events', -> + @EventLogger.checkEventOrder(@id_1, @message_1) + @EventLogger.checkEventOrder(@id_2, @message_2) + status = @EventLogger.checkEventOrder(@id_1, @message_1) + expect(status).to.be.undefined + + it 'should flush old entries', -> + @EventLogger.MAX_EVENTS_BEFORE_CLEAN = 10 + @EventLogger.checkEventOrder(@id_1, @message_1) + for i in [1..8] + status = @EventLogger.checkEventOrder(@id_1, @message_1) + expect(status).to.equal "duplicate" + # the next event should flush the old entries aboce + @EventLogger.MAX_STALE_TIME_IN_MS=1000 + tk.freeze(new Date(@start + 5 * 1000)) + # because we flushed the entries this should not be a duplicate + @EventLogger.checkEventOrder('other-1', @message_2) + status = @EventLogger.checkEventOrder(@id_1, @message_1) + expect(status).to.be.undefined \ No newline at end of file diff --git a/services/real-time/test/unit/coffee/WebsocketLoadBalancerTests.coffee b/services/real-time/test/unit/coffee/WebsocketLoadBalancerTests.coffee index 5cae81a31d..ad0c70832c 100644 --- a/services/real-time/test/unit/coffee/WebsocketLoadBalancerTests.coffee +++ b/services/real-time/test/unit/coffee/WebsocketLoadBalancerTests.coffee @@ -12,6 +12,7 @@ describe "WebsocketLoadBalancer", -> "logger-sharelatex": @logger = { log: sinon.stub(), error: sinon.stub() } "./SafeJsonParse": @SafeJsonParse = parse: (data, cb) => cb null, JSON.parse(data) + "./EventLogger": {checkEventOrder: sinon.stub()} @io = {} @WebsocketLoadBalancer.rclientPub = publish: sinon.stub() @WebsocketLoadBalancer.rclientSub = From 9b25374cd34275dc1a6eb3599d24412be8a60e43 Mon Sep 17 00:00:00 2001 From: Brian Gough Date: Thu, 21 Mar 2019 14:48:51 +0000 Subject: [PATCH 188/491] use time-based cleaning of event log --- services/real-time/app/coffee/EventLogger.coffee | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/services/real-time/app/coffee/EventLogger.coffee b/services/real-time/app/coffee/EventLogger.coffee index 773c9dfa9b..bdc91f940a 100644 --- a/services/real-time/app/coffee/EventLogger.coffee +++ b/services/real-time/app/coffee/EventLogger.coffee @@ -5,11 +5,10 @@ logger = require 'logger-sharelatex' EVENT_LOG_COUNTER = {} EVENT_LOG_TIMESTAMP = {} -EVENT_COUNT = 0 +EVENT_LAST_CLEAN_TIMESTAMP = 0 module.exports = EventLogger = - MAX_EVENTS_BEFORE_CLEAN: 100000 MAX_STALE_TIME_IN_MS: 3600 * 1000 checkEventOrder: (message_id, message) -> @@ -35,8 +34,9 @@ module.exports = EventLogger = EVENT_LOG_COUNTER[key] = count EVENT_LOG_TIMESTAMP[key] = now # periodically remove old counts - if (++EVENT_COUNT % EventLogger.MAX_EVENTS_BEFORE_CLEAN) == 0 + if (now - EVENT_LAST_CLEAN_TIMESTAMP) > EventLogger.MAX_STALE_TIME_IN_MS EventLogger._cleanEventStream(now) + EVENT_LAST_CLEAN_TIMESTAMP = now return previous _cleanEventStream: (now) -> From 57c5ec14bdd3885c22ce5f9beadc7a0f85f53982 Mon Sep 17 00:00:00 2001 From: Brian Gough Date: Thu, 21 Mar 2019 14:49:25 +0000 Subject: [PATCH 189/491] check for a valid counter value in event log --- services/real-time/app/coffee/EventLogger.coffee | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/services/real-time/app/coffee/EventLogger.coffee b/services/real-time/app/coffee/EventLogger.coffee index bdc91f940a..29e79740b9 100644 --- a/services/real-time/app/coffee/EventLogger.coffee +++ b/services/real-time/app/coffee/EventLogger.coffee @@ -15,7 +15,7 @@ module.exports = EventLogger = return if typeof(message_id) isnt 'string' [key, count] = message_id.split("-", 2) count = parseInt(count, 10) - if !count # ignore checks if counter is not present + if !(count >= 0)# ignore checks if counter is not present return # store the last count in a hash for each host previous = EventLogger._storeEventCount(key, count) From 8c82faa966658823e77ea10fa23dcdf9a77e9ca7 Mon Sep 17 00:00:00 2001 From: Brian Gough Date: Thu, 21 Mar 2019 14:50:27 +0000 Subject: [PATCH 190/491] check order of messages on applied-ops channel --- services/real-time/app/coffee/DocumentUpdaterController.coffee | 3 +++ 1 file changed, 3 insertions(+) diff --git a/services/real-time/app/coffee/DocumentUpdaterController.coffee b/services/real-time/app/coffee/DocumentUpdaterController.coffee index 8396be263a..b809bd9086 100644 --- a/services/real-time/app/coffee/DocumentUpdaterController.coffee +++ b/services/real-time/app/coffee/DocumentUpdaterController.coffee @@ -3,6 +3,7 @@ settings = require 'settings-sharelatex' redis = require("redis-sharelatex") rclient = redis.createClient(settings.redis.documentupdater) SafeJsonParse = require "./SafeJsonParse" +EventLogger = require "./EventLogger" MESSAGE_SIZE_LOG_LIMIT = 1024 * 1024 # 1Mb @@ -21,6 +22,8 @@ module.exports = DocumentUpdaterController = logger.error {err: error, channel}, "error parsing JSON" return if message.op? + if message._id? + EventLogger.checkEventOrder(message._id, message) DocumentUpdaterController._applyUpdateFromDocumentUpdater(io, message.doc_id, message.op) else if message.error? DocumentUpdaterController._processErrorFromDocumentUpdater(io, message.doc_id, message.error, message) From e91b967bdb3c2188067892cf743b4bbe8debcf5f Mon Sep 17 00:00:00 2001 From: Brian Gough Date: Thu, 21 Mar 2019 14:56:31 +0000 Subject: [PATCH 191/491] use per-channel event metrics --- .../app/coffee/DocumentUpdaterController.coffee | 2 +- services/real-time/app/coffee/EventLogger.coffee | 10 +++++++--- .../real-time/app/coffee/WebsocketLoadBalancer.coffee | 2 +- 3 files changed, 9 insertions(+), 5 deletions(-) diff --git a/services/real-time/app/coffee/DocumentUpdaterController.coffee b/services/real-time/app/coffee/DocumentUpdaterController.coffee index b809bd9086..dcdd8d142c 100644 --- a/services/real-time/app/coffee/DocumentUpdaterController.coffee +++ b/services/real-time/app/coffee/DocumentUpdaterController.coffee @@ -23,7 +23,7 @@ module.exports = DocumentUpdaterController = return if message.op? if message._id? - EventLogger.checkEventOrder(message._id, message) + EventLogger.checkEventOrder("applied-ops", message._id, message) DocumentUpdaterController._applyUpdateFromDocumentUpdater(io, message.doc_id, message.op) else if message.error? DocumentUpdaterController._processErrorFromDocumentUpdater(io, message.doc_id, message.error, message) diff --git a/services/real-time/app/coffee/EventLogger.coffee b/services/real-time/app/coffee/EventLogger.coffee index 29e79740b9..9a52ccb842 100644 --- a/services/real-time/app/coffee/EventLogger.coffee +++ b/services/real-time/app/coffee/EventLogger.coffee @@ -1,4 +1,5 @@ logger = require 'logger-sharelatex' +metrics = require 'metrics-sharelatex' # keep track of message counters to detect duplicate and out of order events # messsage ids have the format "UNIQUEHOSTKEY-COUNTER" @@ -11,7 +12,7 @@ module.exports = EventLogger = MAX_STALE_TIME_IN_MS: 3600 * 1000 - checkEventOrder: (message_id, message) -> + checkEventOrder: (channel, message_id, message) -> return if typeof(message_id) isnt 'string' [key, count] = message_id.split("-", 2) count = parseInt(count, 10) @@ -20,12 +21,15 @@ module.exports = EventLogger = # store the last count in a hash for each host previous = EventLogger._storeEventCount(key, count) if !previous? || count == (previous + 1) + metrics.inc "event.#{channel}.valid" return # order is ok if (count == previous) - logger.error {key:key, previous: previous, count:count, message:message}, "duplicate event" + metrics.inc "event.#{channel}.duplicate" + # logger.error {key:key, previous: previous, count:count, message:message}, "duplicate event" return "duplicate" else - logger.error {key:key, previous: previous, count:count, message:message}, "events out of order" + metrics.inc "event.#{channel}.out-of-order" + # logger.error {key:key, previous: previous, count:count, message:message}, "events out of order" return # out of order _storeEventCount: (key, count) -> diff --git a/services/real-time/app/coffee/WebsocketLoadBalancer.coffee b/services/real-time/app/coffee/WebsocketLoadBalancer.coffee index 13137805fd..eeedb25916 100644 --- a/services/real-time/app/coffee/WebsocketLoadBalancer.coffee +++ b/services/real-time/app/coffee/WebsocketLoadBalancer.coffee @@ -38,7 +38,7 @@ module.exports = WebsocketLoadBalancer = io.sockets.emit(message.message, message.payload...) else if message.room_id? if message._id? - EventLogger.checkEventOrder(message._id, message) + EventLogger.checkEventOrder("editor-events", message._id, message) io.sockets.in(message.room_id).emit(message.message, message.payload...) else if message.health_check? logger.debug {message}, "got health check message in editor events channel" From 1ab5e526993423367ecbc1ad3df2c252d4e90689 Mon Sep 17 00:00:00 2001 From: Brian Gough Date: Thu, 21 Mar 2019 15:52:53 +0000 Subject: [PATCH 192/491] down-sample valid events by 1000 --- services/real-time/app/coffee/EventLogger.coffee | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/services/real-time/app/coffee/EventLogger.coffee b/services/real-time/app/coffee/EventLogger.coffee index 9a52ccb842..cf412c1904 100644 --- a/services/real-time/app/coffee/EventLogger.coffee +++ b/services/real-time/app/coffee/EventLogger.coffee @@ -21,7 +21,7 @@ module.exports = EventLogger = # store the last count in a hash for each host previous = EventLogger._storeEventCount(key, count) if !previous? || count == (previous + 1) - metrics.inc "event.#{channel}.valid" + metrics.inc "event.#{channel}.valid", 0.001 return # order is ok if (count == previous) metrics.inc "event.#{channel}.duplicate" From 695472a8aaad6ec72c2595a55e5c161833fa8894 Mon Sep 17 00:00:00 2001 From: Brian Gough Date: Fri, 22 Mar 2019 11:18:19 +0000 Subject: [PATCH 193/491] fix event id parsing to allow for dashes in keys --- services/real-time/app/coffee/EventLogger.coffee | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/services/real-time/app/coffee/EventLogger.coffee b/services/real-time/app/coffee/EventLogger.coffee index cf412c1904..1852f0d03d 100644 --- a/services/real-time/app/coffee/EventLogger.coffee +++ b/services/real-time/app/coffee/EventLogger.coffee @@ -14,8 +14,9 @@ module.exports = EventLogger = checkEventOrder: (channel, message_id, message) -> return if typeof(message_id) isnt 'string' - [key, count] = message_id.split("-", 2) - count = parseInt(count, 10) + return if !(result = message_id.match(/^(.*)-(\d+)$/)) + key = result[1] + count = parseInt(result[2], 0) if !(count >= 0)# ignore checks if counter is not present return # store the last count in a hash for each host From c5a09fcf0a78fe6adae0744cd1dfb9eefe5abf93 Mon Sep 17 00:00:00 2001 From: Brian Gough Date: Fri, 22 Mar 2019 11:18:34 +0000 Subject: [PATCH 194/491] add comment about downsampling --- services/real-time/app/coffee/EventLogger.coffee | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/services/real-time/app/coffee/EventLogger.coffee b/services/real-time/app/coffee/EventLogger.coffee index 1852f0d03d..bab9dbaa8c 100644 --- a/services/real-time/app/coffee/EventLogger.coffee +++ b/services/real-time/app/coffee/EventLogger.coffee @@ -22,7 +22,7 @@ module.exports = EventLogger = # store the last count in a hash for each host previous = EventLogger._storeEventCount(key, count) if !previous? || count == (previous + 1) - metrics.inc "event.#{channel}.valid", 0.001 + metrics.inc "event.#{channel}.valid", 0.001 # downsample high rate docupdater events return # order is ok if (count == previous) metrics.inc "event.#{channel}.duplicate" From 6c71ae172b7f0759a3c270f755134a5494a05dea Mon Sep 17 00:00:00 2001 From: Brian Gough Date: Fri, 22 Mar 2019 11:19:08 +0000 Subject: [PATCH 195/491] return out of order events for consistency --- services/real-time/app/coffee/EventLogger.coffee | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/services/real-time/app/coffee/EventLogger.coffee b/services/real-time/app/coffee/EventLogger.coffee index bab9dbaa8c..3baf734ead 100644 --- a/services/real-time/app/coffee/EventLogger.coffee +++ b/services/real-time/app/coffee/EventLogger.coffee @@ -31,7 +31,7 @@ module.exports = EventLogger = else metrics.inc "event.#{channel}.out-of-order" # logger.error {key:key, previous: previous, count:count, message:message}, "events out of order" - return # out of order + return "out-of-order" _storeEventCount: (key, count) -> previous = EVENT_LOG_COUNTER[key] From efa83b4cdeabf42c2b38a8c47c4874056bd0e00a Mon Sep 17 00:00:00 2001 From: Brian Gough Date: Fri, 22 Mar 2019 11:19:31 +0000 Subject: [PATCH 196/491] stub out eventlogger in unit test --- .../test/unit/coffee/DocumentUpdaterControllerTests.coffee | 1 + 1 file changed, 1 insertion(+) diff --git a/services/real-time/test/unit/coffee/DocumentUpdaterControllerTests.coffee b/services/real-time/test/unit/coffee/DocumentUpdaterControllerTests.coffee index fdd2f5fd5c..a2aee89cc0 100644 --- a/services/real-time/test/unit/coffee/DocumentUpdaterControllerTests.coffee +++ b/services/real-time/test/unit/coffee/DocumentUpdaterControllerTests.coffee @@ -22,6 +22,7 @@ describe "DocumentUpdaterController", -> @rclient = {} "./SafeJsonParse": @SafeJsonParse = parse: (data, cb) => cb null, JSON.parse(data) + "./EventLogger": @EventLogger = {checkEventOrder: sinon.stub()} describe "listenForUpdatesFromDocumentUpdater", -> beforeEach -> From 936311f1fb2ec043435b95c7d502350d8f07eef6 Mon Sep 17 00:00:00 2001 From: Brian Gough Date: Fri, 22 Mar 2019 11:20:38 +0000 Subject: [PATCH 197/491] fix eventlogger tests to use name with dashes --- .../test/unit/coffee/EventLoggerTests.coffee | 80 ++++++++++++------- 1 file changed, 52 insertions(+), 28 deletions(-) diff --git a/services/real-time/test/unit/coffee/EventLoggerTests.coffee b/services/real-time/test/unit/coffee/EventLoggerTests.coffee index ce955a8e7d..b0ba546067 100644 --- a/services/real-time/test/unit/coffee/EventLoggerTests.coffee +++ b/services/real-time/test/unit/coffee/EventLoggerTests.coffee @@ -11,9 +11,11 @@ describe 'EventLogger', -> tk.freeze(new Date(@start)) @EventLogger = SandboxedModule.require modulePath, requires: "logger-sharelatex": @logger = {error: sinon.stub()} - @id_1 = "abc-1" + "metrics-sharelatex": @metrics = {inc: sinon.stub()} + @channel = "applied-ops" + @id_1 = "random-hostname:abc-1" @message_1 = "message-1" - @id_2 = "abc-2" + @id_2 = "random-hostname:abc-2" @message_2 = "message-2" afterEach -> @@ -21,32 +23,54 @@ describe 'EventLogger', -> describe 'checkEventOrder', -> - it 'should accept events in order', -> - @EventLogger.checkEventOrder(@id_1, @message_1) - status = @EventLogger.checkEventOrder(@id_2, @message_2) - expect(status).to.be.undefined + describe 'when the events are in order', -> + beforeEach -> + @EventLogger.checkEventOrder(@channel, @id_1, @message_1) + @status = @EventLogger.checkEventOrder(@channel, @id_2, @message_2) - it 'should return "duplicate" for the same event', -> - @EventLogger.checkEventOrder(@id_1, @message_1) - status = @EventLogger.checkEventOrder(@id_1, @message_1) - expect(status).to.equal "duplicate" + it 'should accept events in order', -> + expect(@status).to.be.undefined - it 'should log an error for out of order events', -> - @EventLogger.checkEventOrder(@id_1, @message_1) - @EventLogger.checkEventOrder(@id_2, @message_2) - status = @EventLogger.checkEventOrder(@id_1, @message_1) - expect(status).to.be.undefined + it 'should increment the valid event metric', -> + @metrics.inc.calledWith("event.#{@channel}.valid", 1) + .should.equal.true - it 'should flush old entries', -> - @EventLogger.MAX_EVENTS_BEFORE_CLEAN = 10 - @EventLogger.checkEventOrder(@id_1, @message_1) - for i in [1..8] - status = @EventLogger.checkEventOrder(@id_1, @message_1) - expect(status).to.equal "duplicate" - # the next event should flush the old entries aboce - @EventLogger.MAX_STALE_TIME_IN_MS=1000 - tk.freeze(new Date(@start + 5 * 1000)) - # because we flushed the entries this should not be a duplicate - @EventLogger.checkEventOrder('other-1', @message_2) - status = @EventLogger.checkEventOrder(@id_1, @message_1) - expect(status).to.be.undefined \ No newline at end of file + describe 'when there is a duplicate events', -> + beforeEach -> + @EventLogger.checkEventOrder(@channel, @id_1, @message_1) + @status = @EventLogger.checkEventOrder(@channel, @id_1, @message_1) + + it 'should return "duplicate" for the same event', -> + expect(@status).to.equal "duplicate" + + it 'should increment the duplicate event metric', -> + @metrics.inc.calledWith("event.#{@channel}.duplicate", 1) + .should.equal.true + + describe 'when there are out of order events', -> + beforeEach -> + @EventLogger.checkEventOrder(@channel, @id_1, @message_1) + @EventLogger.checkEventOrder(@channel, @id_2, @message_2) + @status = @EventLogger.checkEventOrder(@channel, @id_1, @message_1) + + it 'should return "out-of-order" for the event', -> + expect(@status).to.equal "out-of-order" + + it 'should increment the out-of-order event metric', -> + @metrics.inc.calledWith("event.#{@channel}.out-of-order", 1) + .should.equal.true + + describe 'after MAX_STALE_TIME_IN_MS', -> + it 'should flush old entries', -> + @EventLogger.MAX_EVENTS_BEFORE_CLEAN = 10 + @EventLogger.checkEventOrder(@channel, @id_1, @message_1) + for i in [1..8] + status = @EventLogger.checkEventOrder(@channel, @id_1, @message_1) + expect(status).to.equal "duplicate" + # the next event should flush the old entries aboce + @EventLogger.MAX_STALE_TIME_IN_MS=1000 + tk.freeze(new Date(@start + 5 * 1000)) + # because we flushed the entries this should not be a duplicate + @EventLogger.checkEventOrder(@channel, 'other-1', @message_2) + status = @EventLogger.checkEventOrder(@channel, @id_1, @message_1) + expect(status).to.be.undefined \ No newline at end of file From 893515e83f20d95cd8398320a047ceaefc26f3c3 Mon Sep 17 00:00:00 2001 From: Brian Gough Date: Tue, 9 Apr 2019 14:48:00 +0100 Subject: [PATCH 198/491] handle duplicate entries in io.sockets.clients --- .../app/coffee/DocumentUpdaterController.coffee | 12 +++++++++--- .../app/coffee/WebsocketLoadBalancer.coffee | 7 ++++++- .../coffee/DocumentUpdaterControllerTests.coffee | 4 +++- .../unit/coffee/WebsocketLoadBalancerTests.coffee | 14 ++++++++++---- 4 files changed, 28 insertions(+), 9 deletions(-) diff --git a/services/real-time/app/coffee/DocumentUpdaterController.coffee b/services/real-time/app/coffee/DocumentUpdaterController.coffee index dcdd8d142c..a2bd3f3e77 100644 --- a/services/real-time/app/coffee/DocumentUpdaterController.coffee +++ b/services/real-time/app/coffee/DocumentUpdaterController.coffee @@ -4,6 +4,7 @@ redis = require("redis-sharelatex") rclient = redis.createClient(settings.redis.documentupdater) SafeJsonParse = require "./SafeJsonParse" EventLogger = require "./EventLogger" +metrics = require "metrics-sharelatex" MESSAGE_SIZE_LOG_LIMIT = 1024 * 1024 # 1Mb @@ -31,13 +32,20 @@ module.exports = DocumentUpdaterController = logger.debug {message}, "got health check message in applied ops channel" _applyUpdateFromDocumentUpdater: (io, doc_id, update) -> - for client in io.sockets.clients(doc_id) + clientList = io.sockets.clients(doc_id) + seen = {} + # send messages only to unique clients (due to duplicate entries in io.sockets.clients) + for client in clientList when not seen[client.id] + seen[client.id] = true if client.id == update.meta.source logger.log doc_id: doc_id, version: update.v, source: update.meta?.source, "distributing update to sender" client.emit "otUpdateApplied", v: update.v, doc: update.doc else if !update.dup # Duplicate ops should just be sent back to sending client for acknowledgement logger.log doc_id: doc_id, version: update.v, source: update.meta?.source, client_id: client.id, "distributing update to collaborator" client.emit "otUpdateApplied", update + if Object.keys(seen).length < clientList.length + metrics.inc "socket-io.duplicate-clients", 0.1 + logger.log doc_id: doc_id, socketIoClients: (client.id for client in clientList), "discarded duplicate clients" _processErrorFromDocumentUpdater: (io, doc_id, error, message) -> for client in io.sockets.clients(doc_id) @@ -46,5 +54,3 @@ module.exports = DocumentUpdaterController = client.disconnect() - - diff --git a/services/real-time/app/coffee/WebsocketLoadBalancer.coffee b/services/real-time/app/coffee/WebsocketLoadBalancer.coffee index eeedb25916..4095e36e68 100644 --- a/services/real-time/app/coffee/WebsocketLoadBalancer.coffee +++ b/services/real-time/app/coffee/WebsocketLoadBalancer.coffee @@ -39,7 +39,12 @@ module.exports = WebsocketLoadBalancer = else if message.room_id? if message._id? EventLogger.checkEventOrder("editor-events", message._id, message) - io.sockets.in(message.room_id).emit(message.message, message.payload...) + # send messages only to unique clients (due to duplicate entries in io.sockets.clients) + clientList = io.sockets.clients(message.room_id) + seen = {} + for client in clientList when not seen[client.id] + seen[client.id] = true + client.emit(message.message, message.payload...) else if message.health_check? logger.debug {message}, "got health check message in editor events channel" diff --git a/services/real-time/test/unit/coffee/DocumentUpdaterControllerTests.coffee b/services/real-time/test/unit/coffee/DocumentUpdaterControllerTests.coffee index a2aee89cc0..c104646696 100644 --- a/services/real-time/test/unit/coffee/DocumentUpdaterControllerTests.coffee +++ b/services/real-time/test/unit/coffee/DocumentUpdaterControllerTests.coffee @@ -23,6 +23,7 @@ describe "DocumentUpdaterController", -> "./SafeJsonParse": @SafeJsonParse = parse: (data, cb) => cb null, JSON.parse(data) "./EventLogger": @EventLogger = {checkEventOrder: sinon.stub()} + "metrics-sharelatex": @metrics = {inc: sinon.stub()} describe "listenForUpdatesFromDocumentUpdater", -> beforeEach -> @@ -81,7 +82,7 @@ describe "DocumentUpdaterController", -> v: @version = 42 doc: @doc_id @io.sockets = - clients: sinon.stub().returns([@sourceClient, @otherClients...]) + clients: sinon.stub().returns([@sourceClient, @otherClients..., @sourceClient]) # include a duplicate client describe "normally", -> beforeEach -> @@ -91,6 +92,7 @@ describe "DocumentUpdaterController", -> @sourceClient.emit .calledWith("otUpdateApplied", v: @version, doc: @doc_id) .should.equal true + @sourceClient.emit.calledOnce.should.equal true it "should get the clients connected to the document", -> @io.sockets.clients diff --git a/services/real-time/test/unit/coffee/WebsocketLoadBalancerTests.coffee b/services/real-time/test/unit/coffee/WebsocketLoadBalancerTests.coffee index ad0c70832c..38c317e9fe 100644 --- a/services/real-time/test/unit/coffee/WebsocketLoadBalancerTests.coffee +++ b/services/real-time/test/unit/coffee/WebsocketLoadBalancerTests.coffee @@ -73,18 +73,24 @@ describe "WebsocketLoadBalancer", -> describe "with a designated room", -> beforeEach -> @io.sockets = - in: sinon.stub().returns(emit: @emit = sinon.stub()) + clients: sinon.stub().returns([ + {id: 'client-id-1', emit: @emit1 = sinon.stub()} + {id: 'client-id-2', emit: @emit2 = sinon.stub()} + {id: 'client-id-1', emit: @emit3 = sinon.stub()} # duplicate client + ]) data = JSON.stringify room_id: @room_id message: @message payload: @payload @WebsocketLoadBalancer._processEditorEvent(@io, "editor-events", data) - it "should send the message to all clients in the room", -> - @io.sockets.in + it "should send the message to all (unique) clients in the room", -> + @io.sockets.clients .calledWith(@room_id) .should.equal true - @emit.calledWith(@message, @payload...).should.equal true + @emit1.calledWith(@message, @payload...).should.equal true + @emit2.calledWith(@message, @payload...).should.equal true + @emit3.called.should.equal false # duplicate client should be ignored describe "when emitting to all", -> beforeEach -> From 2a311392541d2d2e41a36e7249757beed5c1a92b Mon Sep 17 00:00:00 2001 From: Brian Gough Date: Thu, 11 Apr 2019 12:53:43 +0100 Subject: [PATCH 199/491] log and skip duplicate events --- .../app/coffee/DocumentUpdaterController.coffee | 9 ++++++++- services/real-time/app/coffee/EventLogger.coffee | 4 ++-- .../real-time/app/coffee/WebsocketLoadBalancer.coffee | 7 ++++++- 3 files changed, 16 insertions(+), 4 deletions(-) diff --git a/services/real-time/app/coffee/DocumentUpdaterController.coffee b/services/real-time/app/coffee/DocumentUpdaterController.coffee index a2bd3f3e77..057649aa9c 100644 --- a/services/real-time/app/coffee/DocumentUpdaterController.coffee +++ b/services/real-time/app/coffee/DocumentUpdaterController.coffee @@ -24,7 +24,9 @@ module.exports = DocumentUpdaterController = return if message.op? if message._id? - EventLogger.checkEventOrder("applied-ops", message._id, message) + status = EventLogger.checkEventOrder("applied-ops", message._id, message) + if status is 'duplicate' + return # skip duplicate events DocumentUpdaterController._applyUpdateFromDocumentUpdater(io, message.doc_id, message.op) else if message.error? DocumentUpdaterController._processErrorFromDocumentUpdater(io, message.doc_id, message.error, message) @@ -33,6 +35,11 @@ module.exports = DocumentUpdaterController = _applyUpdateFromDocumentUpdater: (io, doc_id, update) -> clientList = io.sockets.clients(doc_id) + # avoid unnecessary work if no clients are connected + if clientList.length is 0 + return + # send updates to clients + logger.log doc_id: doc_id, version: update.v, source: update.meta?.source, socketIoClients: (client.id for client in clientList), "distributing updates to clients" seen = {} # send messages only to unique clients (due to duplicate entries in io.sockets.clients) for client in clientList when not seen[client.id] diff --git a/services/real-time/app/coffee/EventLogger.coffee b/services/real-time/app/coffee/EventLogger.coffee index 3baf734ead..3738e4f729 100644 --- a/services/real-time/app/coffee/EventLogger.coffee +++ b/services/real-time/app/coffee/EventLogger.coffee @@ -26,11 +26,11 @@ module.exports = EventLogger = return # order is ok if (count == previous) metrics.inc "event.#{channel}.duplicate" - # logger.error {key:key, previous: previous, count:count, message:message}, "duplicate event" + logger.warn {channel:channel, message_id:message_id}, "duplicate event" return "duplicate" else metrics.inc "event.#{channel}.out-of-order" - # logger.error {key:key, previous: previous, count:count, message:message}, "events out of order" + logger.warn {channel:channel, message_id:message_id, key:key, previous: previous, count:count}, "out of order event" return "out-of-order" _storeEventCount: (key, count) -> diff --git a/services/real-time/app/coffee/WebsocketLoadBalancer.coffee b/services/real-time/app/coffee/WebsocketLoadBalancer.coffee index 4095e36e68..3e973fdb4c 100644 --- a/services/real-time/app/coffee/WebsocketLoadBalancer.coffee +++ b/services/real-time/app/coffee/WebsocketLoadBalancer.coffee @@ -38,9 +38,14 @@ module.exports = WebsocketLoadBalancer = io.sockets.emit(message.message, message.payload...) else if message.room_id? if message._id? - EventLogger.checkEventOrder("editor-events", message._id, message) + status = EventLogger.checkEventOrder("editor-events", message._id, message) + if status is "duplicate" + return # skip duplicate events # send messages only to unique clients (due to duplicate entries in io.sockets.clients) clientList = io.sockets.clients(message.room_id) + # avoid unnecessary work if no clients are connected + return if clientList.length is 0 + logger.log {channel:channel, message: message.message, room_id: message.room_id, message_id: message._id, socketIoClients: (client.id for client in clientList)}, "distributing event to clients" seen = {} for client in clientList when not seen[client.id] seen[client.id] = true From c6225d614e6d3e084df0a358ab7124f808f82e13 Mon Sep 17 00:00:00 2001 From: Brian Gough Date: Thu, 11 Apr 2019 15:00:25 +0100 Subject: [PATCH 200/491] add /debug/events endpoint --- services/real-time/app.coffee | 5 +++++ .../app/coffee/DocumentUpdaterController.coffee | 1 + services/real-time/app/coffee/EventLogger.coffee | 9 +++++++++ .../real-time/app/coffee/WebsocketLoadBalancer.coffee | 1 + 4 files changed, 16 insertions(+) diff --git a/services/real-time/app.coffee b/services/real-time/app.coffee index 89f0cdfb36..f468ad2d49 100644 --- a/services/real-time/app.coffee +++ b/services/real-time/app.coffee @@ -53,6 +53,11 @@ app.get "/", (req, res, next) -> app.get "/status", (req, res, next) -> res.send "real-time-sharelatex is alive" +app.get "/debug/events", (req, res, next) -> + Settings.debugEvents = parseInt(req.query?.count,10) || 20 + logger.log {count: Settings.debugEvents}, "starting debug mode" + res.send "debug mode will log next #{Settings.debugEvents} events" + rclient = require("redis-sharelatex").createClient(Settings.redis.realtime) app.get "/health_check/redis", (req, res, next) -> rclient.healthCheck (error) -> diff --git a/services/real-time/app/coffee/DocumentUpdaterController.coffee b/services/real-time/app/coffee/DocumentUpdaterController.coffee index 057649aa9c..209af3c0bf 100644 --- a/services/real-time/app/coffee/DocumentUpdaterController.coffee +++ b/services/real-time/app/coffee/DocumentUpdaterController.coffee @@ -15,6 +15,7 @@ module.exports = DocumentUpdaterController = listenForUpdatesFromDocumentUpdater: (io) -> rclient.subscribe "applied-ops" rclient.on "message", (channel, message) -> + EventLogger.debugEvent(channel, message) if settings.debugEvents > 0 DocumentUpdaterController._processMessageFromDocumentUpdater(io, channel, message) _processMessageFromDocumentUpdater: (io, channel, message) -> diff --git a/services/real-time/app/coffee/EventLogger.coffee b/services/real-time/app/coffee/EventLogger.coffee index 3738e4f729..332973659b 100644 --- a/services/real-time/app/coffee/EventLogger.coffee +++ b/services/real-time/app/coffee/EventLogger.coffee @@ -1,5 +1,6 @@ logger = require 'logger-sharelatex' metrics = require 'metrics-sharelatex' +settings = require 'settings-sharelatex' # keep track of message counters to detect duplicate and out of order events # messsage ids have the format "UNIQUEHOSTKEY-COUNTER" @@ -8,10 +9,18 @@ EVENT_LOG_COUNTER = {} EVENT_LOG_TIMESTAMP = {} EVENT_LAST_CLEAN_TIMESTAMP = 0 +# counter for debug logs +COUNTER = 0 + module.exports = EventLogger = MAX_STALE_TIME_IN_MS: 3600 * 1000 + debugEvent: (channel, message) -> + if settings.debugEvents > 0 + logger.log {channel:channel, message:message, counter: COUNTER++}, "logging event" + settings.debugEvents-- + checkEventOrder: (channel, message_id, message) -> return if typeof(message_id) isnt 'string' return if !(result = message_id.match(/^(.*)-(\d+)$/)) diff --git a/services/real-time/app/coffee/WebsocketLoadBalancer.coffee b/services/real-time/app/coffee/WebsocketLoadBalancer.coffee index 3e973fdb4c..8fed080b68 100644 --- a/services/real-time/app/coffee/WebsocketLoadBalancer.coffee +++ b/services/real-time/app/coffee/WebsocketLoadBalancer.coffee @@ -27,6 +27,7 @@ module.exports = WebsocketLoadBalancer = listenForEditorEvents: (io) -> @rclientSub.subscribe "editor-events" @rclientSub.on "message", (channel, message) -> + EventLogger.debugEvent(channel, message) if Settings.debugEvents > 0 WebsocketLoadBalancer._processEditorEvent io, channel, message _processEditorEvent: (io, channel, message) -> From 6374a641d58b3c4a6a8b3e90432880a6588b1bda Mon Sep 17 00:00:00 2001 From: Brian Gough Date: Thu, 11 Apr 2019 15:07:42 +0100 Subject: [PATCH 201/491] fix unit tests --- services/real-time/test/unit/coffee/EventLoggerTests.coffee | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/services/real-time/test/unit/coffee/EventLoggerTests.coffee b/services/real-time/test/unit/coffee/EventLoggerTests.coffee index b0ba546067..93350af848 100644 --- a/services/real-time/test/unit/coffee/EventLoggerTests.coffee +++ b/services/real-time/test/unit/coffee/EventLoggerTests.coffee @@ -10,7 +10,7 @@ describe 'EventLogger', -> @start = Date.now() tk.freeze(new Date(@start)) @EventLogger = SandboxedModule.require modulePath, requires: - "logger-sharelatex": @logger = {error: sinon.stub()} + "logger-sharelatex": @logger = {error: sinon.stub(), warn: sinon.stub()} "metrics-sharelatex": @metrics = {inc: sinon.stub()} @channel = "applied-ops" @id_1 = "random-hostname:abc-1" From ef9e97e7d1fcaf2bee9cf2ffdc5d2d754e3fe736 Mon Sep 17 00:00:00 2001 From: Brian Gough Date: Thu, 11 Apr 2019 15:39:28 +0100 Subject: [PATCH 202/491] add metric for applied-ops events --- services/real-time/app/coffee/DocumentUpdaterController.coffee | 1 + 1 file changed, 1 insertion(+) diff --git a/services/real-time/app/coffee/DocumentUpdaterController.coffee b/services/real-time/app/coffee/DocumentUpdaterController.coffee index 209af3c0bf..c4367cd652 100644 --- a/services/real-time/app/coffee/DocumentUpdaterController.coffee +++ b/services/real-time/app/coffee/DocumentUpdaterController.coffee @@ -15,6 +15,7 @@ module.exports = DocumentUpdaterController = listenForUpdatesFromDocumentUpdater: (io) -> rclient.subscribe "applied-ops" rclient.on "message", (channel, message) -> + metrics.inc "rclient", 0.001 # global event rate metric EventLogger.debugEvent(channel, message) if settings.debugEvents > 0 DocumentUpdaterController._processMessageFromDocumentUpdater(io, channel, message) From cefdd15c5e942a4c1a3a3229a0e72e132c074abd Mon Sep 17 00:00:00 2001 From: Brian Gough Date: Thu, 11 Apr 2019 15:49:19 +0100 Subject: [PATCH 203/491] update to redis-sharelatex v1.0.6 for latest ioredis --- services/real-time/npm-shrinkwrap.json | 24 ++++++++++++------------ services/real-time/package.json | 2 +- 2 files changed, 13 insertions(+), 13 deletions(-) diff --git a/services/real-time/npm-shrinkwrap.json b/services/real-time/npm-shrinkwrap.json index 3706c6be34..14fe405006 100644 --- a/services/real-time/npm-shrinkwrap.json +++ b/services/real-time/npm-shrinkwrap.json @@ -488,9 +488,9 @@ "optional": true }, "denque": { - "version": "1.4.0", + "version": "1.4.1", "from": "denque@>=1.1.0 <2.0.0", - "resolved": "https://registry.npmjs.org/denque/-/denque-1.4.0.tgz" + "resolved": "https://registry.npmjs.org/denque/-/denque-1.4.1.tgz" }, "depd": { "version": "1.1.2", @@ -868,9 +868,9 @@ "resolved": "https://registry.npmjs.org/inherits/-/inherits-2.0.3.tgz" }, "ioredis": { - "version": "4.6.0", - "from": "ioredis@4.6.0", - "resolved": "https://registry.npmjs.org/ioredis/-/ioredis-4.6.0.tgz", + "version": "4.6.3", + "from": "ioredis@>=4.6.0 <4.7.0", + "resolved": "https://registry.npmjs.org/ioredis/-/ioredis-4.6.3.tgz", "dependencies": { "debug": { "version": "3.2.6", @@ -1443,14 +1443,14 @@ } }, "redis-sharelatex": { - "version": "1.0.5", - "from": "https://registry.npmjs.org/redis-sharelatex/-/redis-sharelatex-1.0.5.tgz", - "resolved": "https://registry.npmjs.org/redis-sharelatex/-/redis-sharelatex-1.0.5.tgz", + "version": "1.0.6", + "from": "git+https://github.com/sharelatex/redis-sharelatex.git#v1.0.6", + "resolved": "git+https://github.com/sharelatex/redis-sharelatex.git#0992345e17d204066654260ad718170bbd1018d6", "dependencies": { "async": { - "version": "2.6.1", + "version": "2.6.2", "from": "async@>=2.5.0 <3.0.0", - "resolved": "https://registry.npmjs.org/async/-/async-2.6.1.tgz" + "resolved": "https://registry.npmjs.org/async/-/async-2.6.2.tgz" }, "coffee-script": { "version": "1.8.0", @@ -1665,9 +1665,9 @@ "resolved": "https://registry.npmjs.org/stack-trace/-/stack-trace-0.0.9.tgz" }, "standard-as-callback": { - "version": "1.0.1", + "version": "1.0.2", "from": "standard-as-callback@>=1.0.0 <2.0.0", - "resolved": "https://registry.npmjs.org/standard-as-callback/-/standard-as-callback-1.0.1.tgz" + "resolved": "https://registry.npmjs.org/standard-as-callback/-/standard-as-callback-1.0.2.tgz" }, "statsd-parser": { "version": "0.0.4", diff --git a/services/real-time/package.json b/services/real-time/package.json index ea519b24fc..753a9fc6b8 100644 --- a/services/real-time/package.json +++ b/services/real-time/package.json @@ -30,7 +30,7 @@ "express-session": "^1.9.1", "logger-sharelatex": "^1.6.0", "metrics-sharelatex": "^2.1.1", - "redis-sharelatex": "https://registry.npmjs.org/redis-sharelatex/-/redis-sharelatex-1.0.5.tgz", + "redis-sharelatex": "git+https://github.com/sharelatex/redis-sharelatex.git#v1.0.6", "request": "~2.34.0", "session.socket.io": "^0.1.6", "settings-sharelatex": "^1.1.0", From e72acacf17183aa24bd2aeebc6575c36fcd9dc01 Mon Sep 17 00:00:00 2001 From: Brian Gough Date: Thu, 11 Apr 2019 16:25:42 +0100 Subject: [PATCH 204/491] downsample logging --- services/real-time/app/coffee/EventLogger.coffee | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/services/real-time/app/coffee/EventLogger.coffee b/services/real-time/app/coffee/EventLogger.coffee index 332973659b..5e727759d6 100644 --- a/services/real-time/app/coffee/EventLogger.coffee +++ b/services/real-time/app/coffee/EventLogger.coffee @@ -35,11 +35,12 @@ module.exports = EventLogger = return # order is ok if (count == previous) metrics.inc "event.#{channel}.duplicate" - logger.warn {channel:channel, message_id:message_id}, "duplicate event" + if Math.random() < 0.01 + logger.warn {channel:channel, message_id:message_id}, "duplicate event (sampled at 1%)" return "duplicate" else metrics.inc "event.#{channel}.out-of-order" - logger.warn {channel:channel, message_id:message_id, key:key, previous: previous, count:count}, "out of order event" + # logger.warn {channel:channel, message_id:message_id, key:key, previous: previous, count:count}, "out of order event" return "out-of-order" _storeEventCount: (key, count) -> From 2dbdcf5bc81f450b251322377f47b64f88f7f74a Mon Sep 17 00:00:00 2001 From: Brian Gough Date: Mon, 15 Apr 2019 14:05:26 +0100 Subject: [PATCH 205/491] add health check to pubsub channels --- services/real-time/app.coffee | 10 +++- .../coffee/DocumentUpdaterController.coffee | 2 + .../app/coffee/HealthCheckManager.coffee | 49 +++++++++++++++++++ .../app/coffee/WebsocketLoadBalancer.coffee | 2 + .../DocumentUpdaterControllerTests.coffee | 1 + .../coffee/WebsocketLoadBalancerTests.coffee | 1 + 6 files changed, 63 insertions(+), 2 deletions(-) create mode 100644 services/real-time/app/coffee/HealthCheckManager.coffee diff --git a/services/real-time/app.coffee b/services/real-time/app.coffee index f468ad2d49..6161e63a46 100644 --- a/services/real-time/app.coffee +++ b/services/real-time/app.coffee @@ -20,6 +20,7 @@ SessionSockets = require('session.socket.io') CookieParser = require("cookie-parser") DrainManager = require("./app/js/DrainManager") +HealthCheckManager = require("./app/js/HealthCheckManager") # Set up socket.io server app = express() @@ -64,6 +65,10 @@ app.get "/health_check/redis", (req, res, next) -> if error? logger.err {err: error}, "failed redis health check" res.sendStatus 500 + else if HealthCheckManager.isFailing() + status = HealthCheckManager.status() + logger.err {pubSubErrors: status}, "failed pubsub health check" + res.sendStatus 500 else res.sendStatus 200 @@ -130,8 +135,9 @@ if Settings.continualPubsubTraffic pubSubClient = redis.createClient(Settings.redis.documentupdater) publishJob = (channel, cb)-> - json = JSON.stringify({health_check:true, date: new Date().toString()}) + checker = new HealthCheckManager(channel) logger.debug {channel:channel}, "sending pub to keep connection alive" + json = JSON.stringify({health_check:true, key: checker.id, date: new Date().toString()}) pubSubClient.publish channel, json, (err)-> if err? logger.err {err, channel}, "error publishing pubsub traffic to redis" @@ -139,7 +145,7 @@ if Settings.continualPubsubTraffic runPubSubTraffic = -> async.map ["applied-ops", "editor-events"], publishJob, (err)-> - setTimeout(runPubSubTraffic, 1000 * 60) + setTimeout(runPubSubTraffic, 1000 * 20) runPubSubTraffic() diff --git a/services/real-time/app/coffee/DocumentUpdaterController.coffee b/services/real-time/app/coffee/DocumentUpdaterController.coffee index c4367cd652..1eb5d21274 100644 --- a/services/real-time/app/coffee/DocumentUpdaterController.coffee +++ b/services/real-time/app/coffee/DocumentUpdaterController.coffee @@ -4,6 +4,7 @@ redis = require("redis-sharelatex") rclient = redis.createClient(settings.redis.documentupdater) SafeJsonParse = require "./SafeJsonParse" EventLogger = require "./EventLogger" +HealthCheckManager = require "./HealthCheckManager" metrics = require "metrics-sharelatex" MESSAGE_SIZE_LOG_LIMIT = 1024 * 1024 # 1Mb @@ -34,6 +35,7 @@ module.exports = DocumentUpdaterController = DocumentUpdaterController._processErrorFromDocumentUpdater(io, message.doc_id, message.error, message) else if message.health_check? logger.debug {message}, "got health check message in applied ops channel" + HealthCheckManager.check channel, message.key _applyUpdateFromDocumentUpdater: (io, doc_id, update) -> clientList = io.sockets.clients(doc_id) diff --git a/services/real-time/app/coffee/HealthCheckManager.coffee b/services/real-time/app/coffee/HealthCheckManager.coffee new file mode 100644 index 0000000000..cd6b617b9b --- /dev/null +++ b/services/real-time/app/coffee/HealthCheckManager.coffee @@ -0,0 +1,49 @@ +metrics = require "metrics-sharelatex" + +os = require "os" +HOST = os.hostname() +PID = process.pid +COUNT = 0 + +CHANNEL_MANAGER = {} # hash of event checkers by channel name +CHANNEL_ERROR = {} # error status by channel name + +module.exports = class HealthCheckManager + # create an instance of this class which checks that an event with a unique + # id is received only once within a timeout + constructor: (@channel, timeout = 1000) -> + # unique event string + @id = "host=#{HOST}:pid=#{PID}:count=#{COUNT++}" + # count of number of times the event is received + @count = 0 + # after a timeout check the status of the count + @handler = setTimeout () => + @setStatus() + , timeout + # use a timer to record the latency of the channel + @timer = new metrics.Timer("event.#{@channel}.latency") + # keep a record of these objects to dispatch on + CHANNEL_MANAGER[@channel] = @ + processEvent: (id) -> + # if this is our event record it + if id == @id + @count++ + @timer?.done() + @timer = null # only time the latency of the first event + setStatus: () -> + # if we saw the event anything other than a single time that is an error + error = (@count != 1) + CHANNEL_ERROR[@channel] = error + + # class methods + @check: (channel, id) -> + # dispatch event to manager for channel + CHANNEL_MANAGER[channel]?.processEvent id + @status: () -> + # return status of all channels for logging + return CHANNEL_ERROR + @isFailing: () -> + # check if any channel status is bad + for channel, error of CHANNEL_ERROR + return true if error is true + return false diff --git a/services/real-time/app/coffee/WebsocketLoadBalancer.coffee b/services/real-time/app/coffee/WebsocketLoadBalancer.coffee index 8fed080b68..ffe6e820ca 100644 --- a/services/real-time/app/coffee/WebsocketLoadBalancer.coffee +++ b/services/real-time/app/coffee/WebsocketLoadBalancer.coffee @@ -5,6 +5,7 @@ SafeJsonParse = require "./SafeJsonParse" rclientPub = redis.createClient(Settings.redis.realtime) rclientSub = redis.createClient(Settings.redis.realtime) EventLogger = require "./EventLogger" +HealthCheckManager = require "./HealthCheckManager" module.exports = WebsocketLoadBalancer = rclientPub: rclientPub @@ -53,4 +54,5 @@ module.exports = WebsocketLoadBalancer = client.emit(message.message, message.payload...) else if message.health_check? logger.debug {message}, "got health check message in editor events channel" + HealthCheckManager.check channel, message.key diff --git a/services/real-time/test/unit/coffee/DocumentUpdaterControllerTests.coffee b/services/real-time/test/unit/coffee/DocumentUpdaterControllerTests.coffee index c104646696..5ed274b36c 100644 --- a/services/real-time/test/unit/coffee/DocumentUpdaterControllerTests.coffee +++ b/services/real-time/test/unit/coffee/DocumentUpdaterControllerTests.coffee @@ -23,6 +23,7 @@ describe "DocumentUpdaterController", -> "./SafeJsonParse": @SafeJsonParse = parse: (data, cb) => cb null, JSON.parse(data) "./EventLogger": @EventLogger = {checkEventOrder: sinon.stub()} + "./HealthCheckManager": {check: sinon.stub()} "metrics-sharelatex": @metrics = {inc: sinon.stub()} describe "listenForUpdatesFromDocumentUpdater", -> diff --git a/services/real-time/test/unit/coffee/WebsocketLoadBalancerTests.coffee b/services/real-time/test/unit/coffee/WebsocketLoadBalancerTests.coffee index 38c317e9fe..e786038ab7 100644 --- a/services/real-time/test/unit/coffee/WebsocketLoadBalancerTests.coffee +++ b/services/real-time/test/unit/coffee/WebsocketLoadBalancerTests.coffee @@ -13,6 +13,7 @@ describe "WebsocketLoadBalancer", -> "./SafeJsonParse": @SafeJsonParse = parse: (data, cb) => cb null, JSON.parse(data) "./EventLogger": {checkEventOrder: sinon.stub()} + "./HealthCheckManager": {check: sinon.stub()} @io = {} @WebsocketLoadBalancer.rclientPub = publish: sinon.stub() @WebsocketLoadBalancer.rclientSub = From 15c7c911f98e0e0b6aaa9a3773038e02e8cfc50c Mon Sep 17 00:00:00 2001 From: Brian Gough Date: Mon, 15 Apr 2019 14:46:58 +0100 Subject: [PATCH 206/491] update request module --- services/real-time/npm-shrinkwrap.json | 152 ++++++++----------------- services/real-time/package.json | 2 +- 2 files changed, 51 insertions(+), 103 deletions(-) diff --git a/services/real-time/npm-shrinkwrap.json b/services/real-time/npm-shrinkwrap.json index 14fe405006..95656f81fd 100644 --- a/services/real-time/npm-shrinkwrap.json +++ b/services/real-time/npm-shrinkwrap.json @@ -203,17 +203,10 @@ "from": "arrify@>=1.0.1 <2.0.0", "resolved": "https://registry.npmjs.org/arrify/-/arrify-1.0.1.tgz" }, - "asn1": { - "version": "0.1.11", - "from": "asn1@0.1.11", - "resolved": "https://registry.npmjs.org/asn1/-/asn1-0.1.11.tgz", - "optional": true - }, "assert-plus": { - "version": "0.1.5", - "from": "assert-plus@>=0.1.5 <0.2.0", - "resolved": "https://registry.npmjs.org/assert-plus/-/assert-plus-0.1.5.tgz", - "optional": true + "version": "1.0.0", + "from": "assert-plus@>=1.0.0 <2.0.0", + "resolved": "https://registry.npmjs.org/assert-plus/-/assert-plus-1.0.0.tgz" }, "assertion-error": { "version": "1.0.0", @@ -237,10 +230,9 @@ "resolved": "https://registry.npmjs.org/asynckit/-/asynckit-0.4.0.tgz" }, "aws-sign2": { - "version": "0.5.0", - "from": "aws-sign2@>=0.5.0 <0.6.0", - "resolved": "https://registry.npmjs.org/aws-sign2/-/aws-sign2-0.5.0.tgz", - "optional": true + "version": "0.7.0", + "from": "aws-sign2@>=0.7.0 <0.8.0", + "resolved": "https://registry.npmjs.org/aws-sign2/-/aws-sign2-0.7.0.tgz" }, "aws4": { "version": "1.8.0", @@ -303,11 +295,6 @@ "from": "body-parser@>=1.12.0 <2.0.0", "resolved": "https://registry.npmjs.org/body-parser/-/body-parser-1.18.3.tgz" }, - "boom": { - "version": "0.4.2", - "from": "boom@>=0.4.0 <0.5.0", - "resolved": "https://registry.npmjs.org/boom/-/boom-0.4.2.tgz" - }, "brace-expansion": { "version": "1.1.11", "from": "brace-expansion@^1.1.7", @@ -374,10 +361,9 @@ "resolved": "https://registry.npmjs.org/coffee-script/-/coffee-script-1.12.4.tgz" }, "combined-stream": { - "version": "0.0.7", - "from": "combined-stream@>=0.0.4 <0.1.0", - "resolved": "https://registry.npmjs.org/combined-stream/-/combined-stream-0.0.7.tgz", - "optional": true + "version": "1.0.7", + "from": "combined-stream@>=1.0.6 <1.1.0", + "resolved": "https://registry.npmjs.org/combined-stream/-/combined-stream-1.0.7.tgz" }, "concat-map": { "version": "0.0.1", @@ -441,18 +427,6 @@ "from": "crc@3.4.4", "resolved": "https://registry.npmjs.org/crc/-/crc-3.4.4.tgz" }, - "cryptiles": { - "version": "0.2.2", - "from": "cryptiles@>=0.2.0 <0.3.0", - "resolved": "https://registry.npmjs.org/cryptiles/-/cryptiles-0.2.2.tgz", - "optional": true - }, - "ctype": { - "version": "0.5.3", - "from": "ctype@0.5.3", - "resolved": "https://registry.npmjs.org/ctype/-/ctype-0.5.3.tgz", - "optional": true - }, "dashdash": { "version": "1.14.1", "from": "dashdash@>=1.12.0 <2.0.0", @@ -482,10 +456,9 @@ "resolved": "https://registry.npmjs.org/delay/-/delay-4.1.0.tgz" }, "delayed-stream": { - "version": "0.0.5", - "from": "delayed-stream@0.0.5", - "resolved": "https://registry.npmjs.org/delayed-stream/-/delayed-stream-0.0.5.tgz", - "optional": true + "version": "1.0.0", + "from": "delayed-stream@>=1.0.0 <1.1.0", + "resolved": "https://registry.npmjs.org/delayed-stream/-/delayed-stream-1.0.0.tgz" }, "denque": { "version": "1.4.1", @@ -701,23 +674,14 @@ } }, "forever-agent": { - "version": "0.5.2", - "from": "forever-agent@>=0.5.0 <0.6.0", - "resolved": "https://registry.npmjs.org/forever-agent/-/forever-agent-0.5.2.tgz" + "version": "0.6.1", + "from": "forever-agent@>=0.6.1 <0.7.0", + "resolved": "https://registry.npmjs.org/forever-agent/-/forever-agent-0.6.1.tgz" }, "form-data": { - "version": "0.1.4", - "from": "form-data@>=0.1.0 <0.2.0", - "resolved": "https://registry.npmjs.org/form-data/-/form-data-0.1.4.tgz", - "optional": true, - "dependencies": { - "mime": { - "version": "1.2.11", - "from": "mime@>=1.2.11 <1.3.0", - "resolved": "https://registry.npmjs.org/mime/-/mime-1.2.11.tgz", - "optional": true - } - } + "version": "2.3.3", + "from": "form-data@>=2.3.2 <2.4.0", + "resolved": "https://registry.npmjs.org/form-data/-/form-data-2.3.3.tgz" }, "forwarded": { "version": "0.1.2", @@ -802,12 +766,6 @@ "from": "har-validator@>=5.1.0 <5.2.0", "resolved": "https://registry.npmjs.org/har-validator/-/har-validator-5.1.3.tgz" }, - "hawk": { - "version": "1.0.0", - "from": "hawk@>=1.0.0 <1.1.0", - "resolved": "https://registry.npmjs.org/hawk/-/hawk-1.0.0.tgz", - "optional": true - }, "he": { "version": "1.1.1", "from": "he@1.1.1", @@ -819,21 +777,15 @@ "from": "hex2dec@>=1.0.1 <2.0.0", "resolved": "https://registry.npmjs.org/hex2dec/-/hex2dec-1.1.2.tgz" }, - "hoek": { - "version": "0.9.1", - "from": "hoek@>=0.9.0 <0.10.0", - "resolved": "https://registry.npmjs.org/hoek/-/hoek-0.9.1.tgz" - }, "http-errors": { "version": "1.6.3", "from": "http-errors@>=1.6.3 <1.7.0", "resolved": "https://registry.npmjs.org/http-errors/-/http-errors-1.6.3.tgz" }, "http-signature": { - "version": "0.10.1", - "from": "http-signature@>=0.10.0 <0.11.0", - "resolved": "https://registry.npmjs.org/http-signature/-/http-signature-0.10.1.tgz", - "optional": true + "version": "1.2.0", + "from": "http-signature@>=1.2.0 <1.3.0", + "resolved": "https://registry.npmjs.org/http-signature/-/http-signature-1.2.0.tgz" }, "https-proxy-agent": { "version": "2.2.1", @@ -1260,10 +1212,9 @@ "resolved": "https://registry.npmjs.org/node-forge/-/node-forge-0.7.6.tgz" }, "oauth-sign": { - "version": "0.3.0", - "from": "oauth-sign@>=0.3.0 <0.4.0", - "resolved": "https://registry.npmjs.org/oauth-sign/-/oauth-sign-0.3.0.tgz", - "optional": true + "version": "0.9.0", + "from": "oauth-sign@>=0.9.0 <0.10.0", + "resolved": "https://registry.npmjs.org/oauth-sign/-/oauth-sign-0.9.0.tgz" }, "on-finished": { "version": "2.3.0", @@ -1465,24 +1416,29 @@ } }, "request": { - "version": "2.34.0", - "from": "request@>=2.34.0 <2.35.0", - "resolved": "https://registry.npmjs.org/request/-/request-2.34.0.tgz", + "version": "2.88.0", + "from": "request@2.88.0", + "resolved": "https://registry.npmjs.org/request/-/request-2.88.0.tgz", "dependencies": { - "mime": { - "version": "1.2.11", - "from": "mime@>=1.2.9 <1.3.0", - "resolved": "https://registry.npmjs.org/mime/-/mime-1.2.11.tgz" + "mime-db": { + "version": "1.38.0", + "from": "mime-db@>=1.38.0 <1.39.0", + "resolved": "https://registry.npmjs.org/mime-db/-/mime-db-1.38.0.tgz" }, - "node-uuid": { - "version": "1.4.8", - "from": "node-uuid@>=1.4.0 <1.5.0", - "resolved": "https://registry.npmjs.org/node-uuid/-/node-uuid-1.4.8.tgz" + "mime-types": { + "version": "2.1.22", + "from": "mime-types@>=2.1.19 <2.2.0", + "resolved": "https://registry.npmjs.org/mime-types/-/mime-types-2.1.22.tgz" }, - "qs": { - "version": "0.6.6", - "from": "qs@>=0.6.0 <0.7.0", - "resolved": "https://registry.npmjs.org/qs/-/qs-0.6.6.tgz" + "safe-buffer": { + "version": "5.1.2", + "from": "safe-buffer@>=5.1.2 <6.0.0", + "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.1.2.tgz" + }, + "uuid": { + "version": "3.3.2", + "from": "uuid@>=3.3.2 <4.0.0", + "resolved": "https://registry.npmjs.org/uuid/-/uuid-3.3.2.tgz" } } }, @@ -1603,12 +1559,6 @@ "resolved": "https://registry.npmjs.org/sinon/-/sinon-1.5.2.tgz", "dev": true }, - "sntp": { - "version": "0.2.4", - "from": "sntp@>=0.2.0 <0.3.0", - "resolved": "https://registry.npmjs.org/sntp/-/sntp-0.2.4.tgz", - "optional": true - }, "socket.io": { "version": "0.9.16", "from": "socket.io@0.9.16", @@ -1733,16 +1683,14 @@ "resolved": "https://registry.npmjs.org/tinycolor/-/tinycolor-0.0.1.tgz" }, "tough-cookie": { - "version": "2.3.4", - "from": "tough-cookie@>=0.12.0", - "resolved": "https://registry.npmjs.org/tough-cookie/-/tough-cookie-2.3.4.tgz", - "optional": true + "version": "2.4.3", + "from": "tough-cookie@>=2.4.3 <2.5.0", + "resolved": "https://registry.npmjs.org/tough-cookie/-/tough-cookie-2.4.3.tgz" }, "tunnel-agent": { - "version": "0.3.0", - "from": "tunnel-agent@>=0.3.0 <0.4.0", - "resolved": "https://registry.npmjs.org/tunnel-agent/-/tunnel-agent-0.3.0.tgz", - "optional": true + "version": "0.6.0", + "from": "tunnel-agent@>=0.6.0 <0.7.0", + "resolved": "https://registry.npmjs.org/tunnel-agent/-/tunnel-agent-0.6.0.tgz" }, "tweetnacl": { "version": "0.14.5", diff --git a/services/real-time/package.json b/services/real-time/package.json index 753a9fc6b8..8c2310672e 100644 --- a/services/real-time/package.json +++ b/services/real-time/package.json @@ -31,7 +31,7 @@ "logger-sharelatex": "^1.6.0", "metrics-sharelatex": "^2.1.1", "redis-sharelatex": "git+https://github.com/sharelatex/redis-sharelatex.git#v1.0.6", - "request": "~2.34.0", + "request": "^2.34.0", "session.socket.io": "^0.1.6", "settings-sharelatex": "^1.1.0", "socket.io": "0.9.16", From 2277872022c34c9885fe20efcc535e35970442a3 Mon Sep 17 00:00:00 2001 From: Brian Gough Date: Mon, 15 Apr 2019 15:26:04 +0100 Subject: [PATCH 207/491] update package.json to current version of request --- services/real-time/package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/services/real-time/package.json b/services/real-time/package.json index 8c2310672e..60a978f3b5 100644 --- a/services/real-time/package.json +++ b/services/real-time/package.json @@ -31,7 +31,7 @@ "logger-sharelatex": "^1.6.0", "metrics-sharelatex": "^2.1.1", "redis-sharelatex": "git+https://github.com/sharelatex/redis-sharelatex.git#v1.0.6", - "request": "^2.34.0", + "request": "^2.88.0", "session.socket.io": "^0.1.6", "settings-sharelatex": "^1.1.0", "socket.io": "0.9.16", From 6601e94db7876e0eb2aba9ea2e01fde92615d1c1 Mon Sep 17 00:00:00 2001 From: Jakob Ackermann Date: Tue, 30 Apr 2019 23:33:59 +0200 Subject: [PATCH 208/491] [misc] bump socket.io to 0.9.19 for node7+ support Signed-off-by: Jakob Ackermann --- services/real-time/npm-shrinkwrap.json | 6 +++--- services/real-time/package.json | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/services/real-time/npm-shrinkwrap.json b/services/real-time/npm-shrinkwrap.json index 95656f81fd..a341bea726 100644 --- a/services/real-time/npm-shrinkwrap.json +++ b/services/real-time/npm-shrinkwrap.json @@ -1560,9 +1560,9 @@ "dev": true }, "socket.io": { - "version": "0.9.16", - "from": "socket.io@0.9.16", - "resolved": "https://registry.npmjs.org/socket.io/-/socket.io-0.9.16.tgz", + "version": "0.9.19", + "from": "socket.io@0.9.19", + "resolved": "https://registry.npmjs.org/socket.io/-/socket.io-0.9.19.tgz", "dependencies": { "redis": { "version": "0.7.3", diff --git a/services/real-time/package.json b/services/real-time/package.json index 60a978f3b5..e1e4573696 100644 --- a/services/real-time/package.json +++ b/services/real-time/package.json @@ -34,7 +34,7 @@ "request": "^2.88.0", "session.socket.io": "^0.1.6", "settings-sharelatex": "^1.1.0", - "socket.io": "0.9.16", + "socket.io": "^0.9.19", "socket.io-client": "^0.9.16" }, "devDependencies": { From 79a314d5fd5f8b6a6b3a69371d543496f458b78a Mon Sep 17 00:00:00 2001 From: Jakob Ackermann Date: Wed, 1 May 2019 00:56:32 +0200 Subject: [PATCH 209/491] [misc] disable the flash transport We do not use flash on the website and the policy file provider is not compatible with node7+. Signed-off-by: Jakob Ackermann --- services/real-time/app.coffee | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/services/real-time/app.coffee b/services/real-time/app.coffee index 6161e63a46..5c10bdb36f 100644 --- a/services/real-time/app.coffee +++ b/services/real-time/app.coffee @@ -45,7 +45,7 @@ io.configure -> # gzip uses a Node 0.8.x method of calling the gzip program which # doesn't work with 0.6.x #io.enable('browser client gzip') - io.set('transports', ['websocket', 'flashsocket', 'htmlfile', 'xhr-polling', 'jsonp-polling']) + io.set('transports', ['websocket', 'htmlfile', 'xhr-polling', 'jsonp-polling']) io.set('log level', 1) app.get "/", (req, res, next) -> From 8dc41da0bacb46e8d38311de230ba2ddffc3e8b2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Timoth=C3=A9e=20Alby?= Date: Tue, 7 May 2019 17:45:08 +0100 Subject: [PATCH 210/491] update Git URL in Jenkinsfile --- services/real-time/Jenkinsfile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/services/real-time/Jenkinsfile b/services/real-time/Jenkinsfile index d7b6b353a1..792a7a6afa 100644 --- a/services/real-time/Jenkinsfile +++ b/services/real-time/Jenkinsfile @@ -4,10 +4,10 @@ pipeline { agent any environment { - GIT_PROJECT = "real-time-sharelatex" + GIT_PROJECT = "real-time" JENKINS_WORKFLOW = "real-time-sharelatex" TARGET_URL = "${env.JENKINS_URL}blue/organizations/jenkins/${JENKINS_WORKFLOW}/detail/$BRANCH_NAME/$BUILD_NUMBER/pipeline" - GIT_API_URL = "https://api.github.com/repos/sharelatex/${GIT_PROJECT}/statuses/$GIT_COMMIT" + GIT_API_URL = "https://api.github.com/repos/overleaf/${GIT_PROJECT}/statuses/$GIT_COMMIT" } triggers { From 5f045d9792ed09929ce53516fff3c3cef5618625 Mon Sep 17 00:00:00 2001 From: Christopher Hoskin Date: Mon, 13 May 2019 11:54:37 +0100 Subject: [PATCH 211/491] Update buildscripts to 1.1.21 --- services/real-time/Makefile | 6 ++++-- services/real-time/buildscript.txt | 2 +- services/real-time/docker-compose.ci.yml | 5 ++++- services/real-time/docker-compose.yml | 8 ++++++-- 4 files changed, 15 insertions(+), 6 deletions(-) diff --git a/services/real-time/Makefile b/services/real-time/Makefile index fdf938da5a..573659dd90 100644 --- a/services/real-time/Makefile +++ b/services/real-time/Makefile @@ -1,7 +1,7 @@ # This file was auto-generated, do not edit it directly. # Instead run bin/update_build_scripts from # https://github.com/sharelatex/sharelatex-dev-environment -# Version: 1.1.12 +# Version: 1.1.21 BUILD_NUMBER ?= local BRANCH_NAME ?= $(shell git rev-parse --abbrev-ref HEAD) @@ -26,7 +26,9 @@ test: test_unit test_acceptance test_unit: @[ ! -d test/unit ] && echo "real-time has no unit tests" || $(DOCKER_COMPOSE) run --rm test_unit -test_acceptance: test_clean test_acceptance_pre_run # clear the database before each acceptance test run +test_acceptance: test_clean test_acceptance_pre_run test_acceptance_run + +test_acceptance_run: @[ ! -d test/acceptance ] && echo "real-time has no acceptance tests" || $(DOCKER_COMPOSE) run --rm test_acceptance test_clean: diff --git a/services/real-time/buildscript.txt b/services/real-time/buildscript.txt index 5d6ccb471c..1f61f10934 100644 --- a/services/real-time/buildscript.txt +++ b/services/real-time/buildscript.txt @@ -5,4 +5,4 @@ real-time --dependencies=mongo,redis --docker-repos=gcr.io/overleaf-ops --build-target=docker ---script-version=1.1.12 +--script-version=1.1.21 diff --git a/services/real-time/docker-compose.ci.yml b/services/real-time/docker-compose.ci.yml index 36b52f8f8b..d2bcca9ec6 100644 --- a/services/real-time/docker-compose.ci.yml +++ b/services/real-time/docker-compose.ci.yml @@ -1,7 +1,7 @@ # This file was auto-generated, do not edit it directly. # Instead run bin/update_build_scripts from # https://github.com/sharelatex/sharelatex-dev-environment -# Version: 1.1.12 +# Version: 1.1.21 version: "2" @@ -10,6 +10,8 @@ services: image: ci/$PROJECT_NAME:$BRANCH_NAME-$BUILD_NUMBER user: node command: npm run test:unit:_run + environment: + NODE_ENV: test test_acceptance: @@ -21,6 +23,7 @@ services: MONGO_HOST: mongo POSTGRES_HOST: postgres MOCHA_GREP: ${MOCHA_GREP} + NODE_ENV: test depends_on: - mongo - redis diff --git a/services/real-time/docker-compose.yml b/services/real-time/docker-compose.yml index 8bb7857cb6..47b12619da 100644 --- a/services/real-time/docker-compose.yml +++ b/services/real-time/docker-compose.yml @@ -1,18 +1,19 @@ # This file was auto-generated, do not edit it directly. # Instead run bin/update_build_scripts from # https://github.com/sharelatex/sharelatex-dev-environment -# Version: 1.1.12 +# Version: 1.1.21 version: "2" services: test_unit: - build: . + image: node:6.15.1 volumes: - .:/app working_dir: /app environment: MOCHA_GREP: ${MOCHA_GREP} + NODE_ENV: test command: npm run test:unit user: node @@ -27,6 +28,8 @@ services: MONGO_HOST: mongo POSTGRES_HOST: postgres MOCHA_GREP: ${MOCHA_GREP} + LOG_LEVEL: ERROR + NODE_ENV: test user: node depends_on: - mongo @@ -49,3 +52,4 @@ services: mongo: image: mongo:3.4 + From 20683f309496095df8518d1f8cb0f7daf3d27d8d Mon Sep 17 00:00:00 2001 From: Christopher Hoskin Date: Mon, 13 May 2019 11:55:42 +0100 Subject: [PATCH 212/491] Update Node from 6.15.1 to 10.15.3 --- services/real-time/.nvmrc | 2 +- services/real-time/Dockerfile | 4 ++-- services/real-time/buildscript.txt | 2 +- services/real-time/docker-compose.yml | 2 +- 4 files changed, 5 insertions(+), 5 deletions(-) diff --git a/services/real-time/.nvmrc b/services/real-time/.nvmrc index d36e8d82f3..348076b955 100644 --- a/services/real-time/.nvmrc +++ b/services/real-time/.nvmrc @@ -1 +1 @@ -6.15.1 +10.15.3 diff --git a/services/real-time/Dockerfile b/services/real-time/Dockerfile index e7243c5291..bdfbfbd123 100644 --- a/services/real-time/Dockerfile +++ b/services/real-time/Dockerfile @@ -1,4 +1,4 @@ -FROM node:6.15.1 as app +FROM node:10.15.3 as app WORKDIR /app @@ -12,7 +12,7 @@ COPY . /app RUN npm run compile:all -FROM node:6.15.1 +FROM node:10.15.3 COPY --from=app /app /app diff --git a/services/real-time/buildscript.txt b/services/real-time/buildscript.txt index 1f61f10934..07e0c1e9ac 100644 --- a/services/real-time/buildscript.txt +++ b/services/real-time/buildscript.txt @@ -1,6 +1,6 @@ real-time --language=coffeescript ---node-version=6.15.1 +--node-version=10.15.3 --acceptance-creds=None --dependencies=mongo,redis --docker-repos=gcr.io/overleaf-ops diff --git a/services/real-time/docker-compose.yml b/services/real-time/docker-compose.yml index 47b12619da..0d55e99a9c 100644 --- a/services/real-time/docker-compose.yml +++ b/services/real-time/docker-compose.yml @@ -7,7 +7,7 @@ version: "2" services: test_unit: - image: node:6.15.1 + image: node:10.15.3 volumes: - .:/app working_dir: /app From 20d5cc69a49769e1d2a6b939f9139811d80d660d Mon Sep 17 00:00:00 2001 From: Brian Gough Date: Fri, 24 May 2019 10:19:02 +0100 Subject: [PATCH 213/491] filter invalid updates --- .../app/coffee/DocumentUpdaterManager.coffee | 3 +++ .../coffee/DocumentUpdaterManagerTests.coffee | 19 ++++++++++++++++--- 2 files changed, 19 insertions(+), 3 deletions(-) diff --git a/services/real-time/app/coffee/DocumentUpdaterManager.coffee b/services/real-time/app/coffee/DocumentUpdaterManager.coffee index e183d11cec..9f516c2f2d 100644 --- a/services/real-time/app/coffee/DocumentUpdaterManager.coffee +++ b/services/real-time/app/coffee/DocumentUpdaterManager.coffee @@ -1,4 +1,5 @@ request = require "request" +_ = require "underscore" logger = require "logger-sharelatex" settings = require "settings-sharelatex" metrics = require("metrics-sharelatex") @@ -53,6 +54,8 @@ module.exports = DocumentUpdaterManager = return callback(err) queueChange: (project_id, doc_id, change, callback = ()->)-> + allowedKeys = [ 'doc', 'op', 'v', 'dupIfSource', 'meta', 'lastV', 'hash'] + change = _.pick change, allowedKeys jsonChange = JSON.stringify change if jsonChange.indexOf("\u0000") != -1 error = new Error("null bytes found in op") diff --git a/services/real-time/test/unit/coffee/DocumentUpdaterManagerTests.coffee b/services/real-time/test/unit/coffee/DocumentUpdaterManagerTests.coffee index 2d3e0ad3af..4f2c5d9b51 100644 --- a/services/real-time/test/unit/coffee/DocumentUpdaterManagerTests.coffee +++ b/services/real-time/test/unit/coffee/DocumentUpdaterManagerTests.coffee @@ -122,9 +122,9 @@ describe 'DocumentUpdaterManager', -> describe 'queueChange', -> beforeEach -> @change = { - "action":"removeText", - "range":{"start":{"row":2,"column":2},"end":{"row":2,"column":3}}, - "text":"e" + "doc":"1234567890", + "op":["d":"test", "p":345] + "v": 789 } @rclient.rpush = sinon.stub().yields() @callback = sinon.stub() @@ -161,3 +161,16 @@ describe 'DocumentUpdaterManager', -> it "should not push the change onto the pending-updates-list queue", -> @rclient.rpush.called.should.equal false + + describe "with invalid keys", -> + beforeEach -> + @change = { + "op":["d":"test", "p":345] + "version": 789 # not a valid key + } + @DocumentUpdaterManager.queueChange(@project_id, @doc_id, @change, @callback) + + it "should remove the invalid keys from the change", -> + @rclient.rpush + .calledWith("PendingUpdates:#{@doc_id}", JSON.stringify({op:@change.op})) + .should.equal true From 78372119f81390f6c74fd31001bd9f4a033d2d3b Mon Sep 17 00:00:00 2001 From: Henry Oswald Date: Fri, 24 May 2019 15:21:48 +0100 Subject: [PATCH 214/491] Revert "Csh issue 1118 node 10.15.3" --- services/real-time/.nvmrc | 2 +- services/real-time/Dockerfile | 4 ++-- services/real-time/Makefile | 6 ++---- services/real-time/app.coffee | 2 +- services/real-time/buildscript.txt | 4 ++-- services/real-time/docker-compose.ci.yml | 5 +---- services/real-time/docker-compose.yml | 8 ++------ services/real-time/npm-shrinkwrap.json | 6 +++--- services/real-time/package.json | 2 +- 9 files changed, 15 insertions(+), 24 deletions(-) diff --git a/services/real-time/.nvmrc b/services/real-time/.nvmrc index 348076b955..d36e8d82f3 100644 --- a/services/real-time/.nvmrc +++ b/services/real-time/.nvmrc @@ -1 +1 @@ -10.15.3 +6.15.1 diff --git a/services/real-time/Dockerfile b/services/real-time/Dockerfile index bdfbfbd123..e7243c5291 100644 --- a/services/real-time/Dockerfile +++ b/services/real-time/Dockerfile @@ -1,4 +1,4 @@ -FROM node:10.15.3 as app +FROM node:6.15.1 as app WORKDIR /app @@ -12,7 +12,7 @@ COPY . /app RUN npm run compile:all -FROM node:10.15.3 +FROM node:6.15.1 COPY --from=app /app /app diff --git a/services/real-time/Makefile b/services/real-time/Makefile index 573659dd90..fdf938da5a 100644 --- a/services/real-time/Makefile +++ b/services/real-time/Makefile @@ -1,7 +1,7 @@ # This file was auto-generated, do not edit it directly. # Instead run bin/update_build_scripts from # https://github.com/sharelatex/sharelatex-dev-environment -# Version: 1.1.21 +# Version: 1.1.12 BUILD_NUMBER ?= local BRANCH_NAME ?= $(shell git rev-parse --abbrev-ref HEAD) @@ -26,9 +26,7 @@ test: test_unit test_acceptance test_unit: @[ ! -d test/unit ] && echo "real-time has no unit tests" || $(DOCKER_COMPOSE) run --rm test_unit -test_acceptance: test_clean test_acceptance_pre_run test_acceptance_run - -test_acceptance_run: +test_acceptance: test_clean test_acceptance_pre_run # clear the database before each acceptance test run @[ ! -d test/acceptance ] && echo "real-time has no acceptance tests" || $(DOCKER_COMPOSE) run --rm test_acceptance test_clean: diff --git a/services/real-time/app.coffee b/services/real-time/app.coffee index 5c10bdb36f..6161e63a46 100644 --- a/services/real-time/app.coffee +++ b/services/real-time/app.coffee @@ -45,7 +45,7 @@ io.configure -> # gzip uses a Node 0.8.x method of calling the gzip program which # doesn't work with 0.6.x #io.enable('browser client gzip') - io.set('transports', ['websocket', 'htmlfile', 'xhr-polling', 'jsonp-polling']) + io.set('transports', ['websocket', 'flashsocket', 'htmlfile', 'xhr-polling', 'jsonp-polling']) io.set('log level', 1) app.get "/", (req, res, next) -> diff --git a/services/real-time/buildscript.txt b/services/real-time/buildscript.txt index 07e0c1e9ac..5d6ccb471c 100644 --- a/services/real-time/buildscript.txt +++ b/services/real-time/buildscript.txt @@ -1,8 +1,8 @@ real-time --language=coffeescript ---node-version=10.15.3 +--node-version=6.15.1 --acceptance-creds=None --dependencies=mongo,redis --docker-repos=gcr.io/overleaf-ops --build-target=docker ---script-version=1.1.21 +--script-version=1.1.12 diff --git a/services/real-time/docker-compose.ci.yml b/services/real-time/docker-compose.ci.yml index d2bcca9ec6..36b52f8f8b 100644 --- a/services/real-time/docker-compose.ci.yml +++ b/services/real-time/docker-compose.ci.yml @@ -1,7 +1,7 @@ # This file was auto-generated, do not edit it directly. # Instead run bin/update_build_scripts from # https://github.com/sharelatex/sharelatex-dev-environment -# Version: 1.1.21 +# Version: 1.1.12 version: "2" @@ -10,8 +10,6 @@ services: image: ci/$PROJECT_NAME:$BRANCH_NAME-$BUILD_NUMBER user: node command: npm run test:unit:_run - environment: - NODE_ENV: test test_acceptance: @@ -23,7 +21,6 @@ services: MONGO_HOST: mongo POSTGRES_HOST: postgres MOCHA_GREP: ${MOCHA_GREP} - NODE_ENV: test depends_on: - mongo - redis diff --git a/services/real-time/docker-compose.yml b/services/real-time/docker-compose.yml index 0d55e99a9c..8bb7857cb6 100644 --- a/services/real-time/docker-compose.yml +++ b/services/real-time/docker-compose.yml @@ -1,19 +1,18 @@ # This file was auto-generated, do not edit it directly. # Instead run bin/update_build_scripts from # https://github.com/sharelatex/sharelatex-dev-environment -# Version: 1.1.21 +# Version: 1.1.12 version: "2" services: test_unit: - image: node:10.15.3 + build: . volumes: - .:/app working_dir: /app environment: MOCHA_GREP: ${MOCHA_GREP} - NODE_ENV: test command: npm run test:unit user: node @@ -28,8 +27,6 @@ services: MONGO_HOST: mongo POSTGRES_HOST: postgres MOCHA_GREP: ${MOCHA_GREP} - LOG_LEVEL: ERROR - NODE_ENV: test user: node depends_on: - mongo @@ -52,4 +49,3 @@ services: mongo: image: mongo:3.4 - diff --git a/services/real-time/npm-shrinkwrap.json b/services/real-time/npm-shrinkwrap.json index a341bea726..95656f81fd 100644 --- a/services/real-time/npm-shrinkwrap.json +++ b/services/real-time/npm-shrinkwrap.json @@ -1560,9 +1560,9 @@ "dev": true }, "socket.io": { - "version": "0.9.19", - "from": "socket.io@0.9.19", - "resolved": "https://registry.npmjs.org/socket.io/-/socket.io-0.9.19.tgz", + "version": "0.9.16", + "from": "socket.io@0.9.16", + "resolved": "https://registry.npmjs.org/socket.io/-/socket.io-0.9.16.tgz", "dependencies": { "redis": { "version": "0.7.3", diff --git a/services/real-time/package.json b/services/real-time/package.json index e1e4573696..60a978f3b5 100644 --- a/services/real-time/package.json +++ b/services/real-time/package.json @@ -34,7 +34,7 @@ "request": "^2.88.0", "session.socket.io": "^0.1.6", "settings-sharelatex": "^1.1.0", - "socket.io": "^0.9.19", + "socket.io": "0.9.16", "socket.io-client": "^0.9.16" }, "devDependencies": { From 74db743ffa7ac215b21ce351de1e63c6f571242a Mon Sep 17 00:00:00 2001 From: Brian Gough Date: Fri, 24 May 2019 15:23:01 +0100 Subject: [PATCH 215/491] allow fractional drain rate --- services/real-time/app/coffee/DrainManager.coffee | 9 ++++++++- services/real-time/app/coffee/HttpApiController.coffee | 2 +- 2 files changed, 9 insertions(+), 2 deletions(-) diff --git a/services/real-time/app/coffee/DrainManager.coffee b/services/real-time/app/coffee/DrainManager.coffee index 92b59b0751..da6b331e23 100644 --- a/services/real-time/app/coffee/DrainManager.coffee +++ b/services/real-time/app/coffee/DrainManager.coffee @@ -6,9 +6,16 @@ module.exports = clearInterval @interval if rate == 0 return + else if rate < 1 + # allow lower drain rates + # e.g. rate=0.1 will drain one client every 10 seconds + pollingInterval = 1000 / rate + rate = 1 + else + pollingInterval = 1000 @interval = setInterval () => @reconnectNClients(io, rate) - , 1000 + , pollingInterval RECONNECTED_CLIENTS: {} reconnectNClients: (io, N) -> diff --git a/services/real-time/app/coffee/HttpApiController.coffee b/services/real-time/app/coffee/HttpApiController.coffee index a2a9d4d23c..f99bef6dd6 100644 --- a/services/real-time/app/coffee/HttpApiController.coffee +++ b/services/real-time/app/coffee/HttpApiController.coffee @@ -15,7 +15,7 @@ module.exports = HttpApiController = startDrain: (req, res, next) -> io = req.app.get("io") rate = req.query.rate or "4" - rate = parseInt(rate, 10) + rate = parseFloat(rate) || 0 logger.log {rate}, "setting client drain rate" DrainManager.startDrain io, rate res.send 204 \ No newline at end of file From 47e0cb44cedfe8bff8391bf297c49b9f505fbbe2 Mon Sep 17 00:00:00 2001 From: Henry Oswald Date: Thu, 30 May 2019 10:29:34 +0100 Subject: [PATCH 216/491] bump redis to 1.0.8 --- services/real-time/npm-shrinkwrap.json | 41 ++++++++++++++++++++------ services/real-time/package.json | 2 +- 2 files changed, 33 insertions(+), 10 deletions(-) diff --git a/services/real-time/npm-shrinkwrap.json b/services/real-time/npm-shrinkwrap.json index 95656f81fd..4c0e2a0f85 100644 --- a/services/real-time/npm-shrinkwrap.json +++ b/services/real-time/npm-shrinkwrap.json @@ -1394,24 +1394,47 @@ } }, "redis-sharelatex": { - "version": "1.0.6", - "from": "git+https://github.com/sharelatex/redis-sharelatex.git#v1.0.6", - "resolved": "git+https://github.com/sharelatex/redis-sharelatex.git#0992345e17d204066654260ad718170bbd1018d6", + "version": "1.0.8", + "resolved": "https://registry.npmjs.org/redis-sharelatex/-/redis-sharelatex-1.0.8.tgz", + "integrity": "sha512-X88/tG03NKWoy0uMzTrzARvILaFj9ZoKGhECtjf8N7GeCzo90zCeT0cVIJCVHeECogXCxBRf/ABFUBBQKUOCew==", + "requires": { + "async": "^2.5.0", + "coffee-script": "1.8.0", + "ioredis": "~4.9.1", + "redis-sentinel": "0.1.1", + "underscore": "1.7.0" + }, "dependencies": { "async": { "version": "2.6.2", - "from": "async@>=2.5.0 <3.0.0", - "resolved": "https://registry.npmjs.org/async/-/async-2.6.2.tgz" + "resolved": "https://registry.npmjs.org/async/-/async-2.6.2.tgz", + "integrity": "sha512-H1qVYh1MYhEEFLsP97cVKqCGo7KfCyTt6uEWqsTBr9SO84oK9Uwbyd/yCW+6rKJLHksBNUVWZDAjfS+Ccx0Bbg==", + "requires": { + "lodash": "^4.17.11" + } }, "coffee-script": { "version": "1.8.0", - "from": "coffee-script@1.8.0", - "resolved": "https://registry.npmjs.org/coffee-script/-/coffee-script-1.8.0.tgz" + "resolved": "https://registry.npmjs.org/coffee-script/-/coffee-script-1.8.0.tgz", + "integrity": "sha1-nJ8dK0pSoADe0Vtll5FwNkgmPB0=", + "requires": { + "mkdirp": "~0.3.5" + } + }, + "lodash": { + "version": "4.17.11", + "resolved": "https://registry.npmjs.org/lodash/-/lodash-4.17.11.tgz", + "integrity": "sha512-cQKh8igo5QUhZ7lg38DYWAxMvjSAKG0A8wGSVimP07SIUEK2UO+arSRKbRZWtelMtN5V0Hkwh5ryOto/SshYIg==" }, "mkdirp": { "version": "0.3.5", - "from": "mkdirp@>=0.3.5 <0.4.0", - "resolved": "https://registry.npmjs.org/mkdirp/-/mkdirp-0.3.5.tgz" + "resolved": "https://registry.npmjs.org/mkdirp/-/mkdirp-0.3.5.tgz", + "integrity": "sha1-3j5fiWHIjHh+4TaN+EmsRBPsqNc=" + }, + "underscore": { + "version": "1.7.0", + "resolved": "https://registry.npmjs.org/underscore/-/underscore-1.7.0.tgz", + "integrity": "sha1-a7rwh3UA02vjTsqlhODbn+8DUgk=" } } }, diff --git a/services/real-time/package.json b/services/real-time/package.json index 60a978f3b5..3c8a05591a 100644 --- a/services/real-time/package.json +++ b/services/real-time/package.json @@ -30,7 +30,7 @@ "express-session": "^1.9.1", "logger-sharelatex": "^1.6.0", "metrics-sharelatex": "^2.1.1", - "redis-sharelatex": "git+https://github.com/sharelatex/redis-sharelatex.git#v1.0.6", + "redis-sharelatex": "^1.0.8", "request": "^2.88.0", "session.socket.io": "^0.1.6", "settings-sharelatex": "^1.1.0", From 2c9b222437dbb26d3217f52046ce61fb3785ad5b Mon Sep 17 00:00:00 2001 From: Henry Oswald Date: Thu, 30 May 2019 10:58:05 +0100 Subject: [PATCH 217/491] fix breaking tests from using ioredis with inbuilt promises because a promise is returned from ioredis it errors in mocha as it can't take a promise and a callback --- .../test/acceptance/coffee/ApplyUpdateTests.coffee | 9 +++++++-- .../real-time/test/acceptance/coffee/SessionTests.coffee | 5 +++-- 2 files changed, 10 insertions(+), 4 deletions(-) diff --git a/services/real-time/test/acceptance/coffee/ApplyUpdateTests.coffee b/services/real-time/test/acceptance/coffee/ApplyUpdateTests.coffee index d9addc990d..842e4fcc4c 100644 --- a/services/real-time/test/acceptance/coffee/ApplyUpdateTests.coffee +++ b/services/real-time/test/acceptance/coffee/ApplyUpdateTests.coffee @@ -48,6 +48,7 @@ describe "applyOtUpdate", -> rclient.lrange "pending-updates-list", 0, -1, (error, [doc_id]) => doc_id.should.equal "#{@project_id}:#{@doc_id}" done() + return null it "should push the update into redis", (done) -> rclient.lrange redisSettings.documentupdater.key_schema.pendingUpdates({@doc_id}), 0, -1, (error, [update]) => @@ -58,7 +59,8 @@ describe "applyOtUpdate", -> user_id: @user_id } done() - + return null + after (done) -> async.series [ (cb) => rclient.del "pending-updates-list", cb @@ -107,6 +109,7 @@ describe "applyOtUpdate", -> rclient.llen redisSettings.documentupdater.key_schema.pendingUpdates({@doc_id}), (error, len) => len.should.equal 0 done() + return null describe "when authorized to read-only with a comment update", -> before (done) -> @@ -142,6 +145,7 @@ describe "applyOtUpdate", -> rclient.lrange "pending-updates-list", 0, -1, (error, [doc_id]) => doc_id.should.equal "#{@project_id}:#{@doc_id}" done() + return null it "should push the update into redis", (done) -> rclient.lrange redisSettings.documentupdater.key_schema.pendingUpdates({@doc_id}), 0, -1, (error, [update]) => @@ -152,7 +156,8 @@ describe "applyOtUpdate", -> user_id: @user_id } done() - + return null + after (done) -> async.series [ (cb) => rclient.del "pending-updates-list", cb diff --git a/services/real-time/test/acceptance/coffee/SessionTests.coffee b/services/real-time/test/acceptance/coffee/SessionTests.coffee index fa5378a1f5..23c4e78ce9 100644 --- a/services/real-time/test/acceptance/coffee/SessionTests.coffee +++ b/services/real-time/test/acceptance/coffee/SessionTests.coffee @@ -12,8 +12,9 @@ describe "Session", -> }, (error) => throw error if error? @client = RealTimeClient.connect() - done() - + return done() + return null + it "should not get disconnected", (done) -> disconnected = false @client.on "disconnect", () -> From 421a914e7249aeec0b384d832049dd89ec0cd1c3 Mon Sep 17 00:00:00 2001 From: Henry Oswald Date: Fri, 31 May 2019 09:15:49 +0100 Subject: [PATCH 218/491] log out when health check manager fails a check --- services/real-time/app/coffee/HealthCheckManager.coffee | 3 +++ 1 file changed, 3 insertions(+) diff --git a/services/real-time/app/coffee/HealthCheckManager.coffee b/services/real-time/app/coffee/HealthCheckManager.coffee index cd6b617b9b..bcd3e2ed07 100644 --- a/services/real-time/app/coffee/HealthCheckManager.coffee +++ b/services/real-time/app/coffee/HealthCheckManager.coffee @@ -1,4 +1,5 @@ metrics = require "metrics-sharelatex" +logger = require("logger-sharelatex") os = require "os" HOST = os.hostname() @@ -32,6 +33,8 @@ module.exports = class HealthCheckManager @timer = null # only time the latency of the first event setStatus: () -> # if we saw the event anything other than a single time that is an error + if @count != 1 + logger.err channel:@channel, count:@count, id:@id, "redis channel health check error" error = (@count != 1) CHANNEL_ERROR[@channel] = error From acf850bce90c113fb2db26c6518e6a943a9343bb Mon Sep 17 00:00:00 2001 From: Eric Mc Sween Date: Fri, 31 May 2019 17:27:05 -0400 Subject: [PATCH 219/491] Do not log error on doc updater 404 Document updater 404s are not indicative of a problem, but just of the client trying to connect to a deleted document, which is easy to trigger. We log a warning instead. --- .../app/coffee/DocumentUpdaterManager.coffee | 4 +- .../coffee/DocumentUpdaterManagerTests.coffee | 45 ++++++++++--------- 2 files changed, 25 insertions(+), 24 deletions(-) diff --git a/services/real-time/app/coffee/DocumentUpdaterManager.coffee b/services/real-time/app/coffee/DocumentUpdaterManager.coffee index e183d11cec..e21a6a05fa 100644 --- a/services/real-time/app/coffee/DocumentUpdaterManager.coffee +++ b/services/real-time/app/coffee/DocumentUpdaterManager.coffee @@ -23,7 +23,7 @@ module.exports = DocumentUpdaterManager = catch error return callback(error) callback null, body?.lines, body?.version, body?.ranges, body?.ops - else if res.statusCode == 422 # Unprocessable Entity + else if res.statusCode in [404, 422] err = new Error("doc updater could not load requested ops") err.statusCode = res.statusCode logger.warn {err, project_id, doc_id, url, fromVersion}, "doc updater could not load requested ops" @@ -51,7 +51,7 @@ module.exports = DocumentUpdaterManager = err.statusCode = res.statusCode logger.error {err, project_id}, "document updater returned failure status code: #{res.statusCode}" return callback(err) - + queueChange: (project_id, doc_id, change, callback = ()->)-> jsonChange = JSON.stringify change if jsonChange.indexOf("\u0000") != -1 diff --git a/services/real-time/test/unit/coffee/DocumentUpdaterManagerTests.coffee b/services/real-time/test/unit/coffee/DocumentUpdaterManagerTests.coffee index 2d3e0ad3af..e220ce4275 100644 --- a/services/real-time/test/unit/coffee/DocumentUpdaterManagerTests.coffee +++ b/services/real-time/test/unit/coffee/DocumentUpdaterManagerTests.coffee @@ -10,13 +10,13 @@ describe 'DocumentUpdaterManager', -> @doc_id = "doc-id-394" @lines = ["one", "two", "three"] @version = 42 - @settings = + @settings = apis: documentupdater: url: "http://doc-updater.example.com" redis: documentupdater: key_schema: pendingUpdates: ({doc_id}) -> "PendingUpdates:#{doc_id}" @rclient = {auth:->} - + @DocumentUpdaterManager = SandboxedModule.require modulePath, requires: 'settings-sharelatex':@settings @@ -59,17 +59,19 @@ describe 'DocumentUpdaterManager', -> it "should return an error to the callback", -> @callback.calledWith(@error).should.equal true - describe "when the document updater returns a 422 status code", -> - beforeEach -> - @request.get = sinon.stub().callsArgWith(1, null, { statusCode: 422 }, "") - @DocumentUpdaterManager.getDocument @project_id, @doc_id, @fromVersion, @callback + [404, 422].forEach (statusCode) -> + describe "when the document updater returns a #{statusCode} status code", -> + beforeEach -> + @request.get = sinon.stub().callsArgWith(1, null, { statusCode }, "") + @DocumentUpdaterManager.getDocument @project_id, @doc_id, @fromVersion, @callback - it "should return the callback with an error", -> - err = new Error("doc updater could not load requested ops") - err.statusCode = 422 - @callback - .calledWith(err) - .should.equal true + it "should return the callback with an error", -> + @callback.called.should.equal(true) + err = @callback.getCall(0).args[0] + err.should.have.property('statusCode', statusCode) + err.should.have.property('message', "doc updater could not load requested ops") + @logger.error.called.should.equal(false) + @logger.warn.called.should.equal(true) describe "when the document updater returns a failure error code", -> beforeEach -> @@ -77,11 +79,11 @@ describe 'DocumentUpdaterManager', -> @DocumentUpdaterManager.getDocument @project_id, @doc_id, @fromVersion, @callback it "should return the callback with an error", -> - err = new Error("doc updater returned failure status code: 500") - err.statusCode = 500 - @callback - .calledWith(err) - .should.equal true + @callback.called.should.equal(true) + err = @callback.getCall(0).args[0] + err.should.have.property('statusCode', 500) + err.should.have.property('message', "doc updater returned a non-success status code: 500") + @logger.error.called.should.equal(true) describe 'flushProjectToMongoAndDelete', -> beforeEach -> @@ -113,11 +115,10 @@ describe 'DocumentUpdaterManager', -> @DocumentUpdaterManager.flushProjectToMongoAndDelete @project_id, @callback it "should return the callback with an error", -> - err = new Error("doc updater returned failure status code: 500") - err.statusCode = 500 - @callback - .calledWith(err) - .should.equal true + @callback.called.should.equal(true) + err = @callback.getCall(0).args[0] + err.should.have.property('statusCode', 500) + err.should.have.property('message', "document updater returned a failure status code: 500") describe 'queueChange', -> beforeEach -> From 63f052192e2cb31b45cf0656b4d21d0c95a8b62e Mon Sep 17 00:00:00 2001 From: Brian Gough Date: Mon, 3 Jun 2019 09:42:12 +0100 Subject: [PATCH 220/491] use background flush on disconnect --- services/real-time/app/coffee/DocumentUpdaterManager.coffee | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/services/real-time/app/coffee/DocumentUpdaterManager.coffee b/services/real-time/app/coffee/DocumentUpdaterManager.coffee index 9f516c2f2d..d5df28ed49 100644 --- a/services/real-time/app/coffee/DocumentUpdaterManager.coffee +++ b/services/real-time/app/coffee/DocumentUpdaterManager.coffee @@ -36,9 +36,11 @@ module.exports = DocumentUpdaterManager = callback err flushProjectToMongoAndDelete: (project_id, callback = ()->) -> + # this method is called when the last connected user leaves the project logger.log project_id:project_id, "deleting project from document updater" timer = new metrics.Timer("delete.mongo.project") - url = "#{settings.apis.documentupdater.url}/project/#{project_id}" + # flush the project in the background when all users have left + url = "#{settings.apis.documentupdater.url}/project/#{project_id}?background=true" request.del url, (err, res, body)-> timer.done() if err? From 973a25fc644ea885355f5bdc33368cccacea192e Mon Sep 17 00:00:00 2001 From: Henry Oswald Date: Mon, 3 Jun 2019 10:33:32 +0100 Subject: [PATCH 221/491] fix background flush unit test --- .../test/unit/coffee/DocumentUpdaterManagerTests.coffee | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/services/real-time/test/unit/coffee/DocumentUpdaterManagerTests.coffee b/services/real-time/test/unit/coffee/DocumentUpdaterManagerTests.coffee index 4f2c5d9b51..e6d82f24ff 100644 --- a/services/real-time/test/unit/coffee/DocumentUpdaterManagerTests.coffee +++ b/services/real-time/test/unit/coffee/DocumentUpdaterManagerTests.coffee @@ -93,7 +93,7 @@ describe 'DocumentUpdaterManager', -> @DocumentUpdaterManager.flushProjectToMongoAndDelete @project_id, @callback it 'should delete the project from the document updater', -> - url = "#{@settings.apis.documentupdater.url}/project/#{@project_id}" + url = "#{@settings.apis.documentupdater.url}/project/#{@project_id}?background=true" @request.del.calledWith(url).should.equal true it "should call the callback with no error", -> From 40f3456b1c4c82b166dba6cff71bc2e48fd28a7c Mon Sep 17 00:00:00 2001 From: Brian Gough Date: Mon, 3 Jun 2019 10:34:59 +0100 Subject: [PATCH 222/491] update unit test --- .../test/unit/coffee/DocumentUpdaterManagerTests.coffee | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/services/real-time/test/unit/coffee/DocumentUpdaterManagerTests.coffee b/services/real-time/test/unit/coffee/DocumentUpdaterManagerTests.coffee index 4f2c5d9b51..e6d82f24ff 100644 --- a/services/real-time/test/unit/coffee/DocumentUpdaterManagerTests.coffee +++ b/services/real-time/test/unit/coffee/DocumentUpdaterManagerTests.coffee @@ -93,7 +93,7 @@ describe 'DocumentUpdaterManager', -> @DocumentUpdaterManager.flushProjectToMongoAndDelete @project_id, @callback it 'should delete the project from the document updater', -> - url = "#{@settings.apis.documentupdater.url}/project/#{@project_id}" + url = "#{@settings.apis.documentupdater.url}/project/#{@project_id}?background=true" @request.del.calledWith(url).should.equal true it "should call the callback with no error", -> From 46dfe56b05b0ccced45a4b35ddff7262fc7fae93 Mon Sep 17 00:00:00 2001 From: miguel Date: Fri, 21 Jun 2019 07:30:12 +0200 Subject: [PATCH 223/491] Downgraded unathorised log to warning --- services/real-time/app/coffee/WebsocketController.coffee | 2 +- .../test/unit/coffee/WebsocketControllerTests.coffee | 5 ++++- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/services/real-time/app/coffee/WebsocketController.coffee b/services/real-time/app/coffee/WebsocketController.coffee index 96a74b9748..470b1f2a52 100644 --- a/services/real-time/app/coffee/WebsocketController.coffee +++ b/services/real-time/app/coffee/WebsocketController.coffee @@ -22,7 +22,7 @@ module.exports = WebsocketController = if !privilegeLevel or privilegeLevel == "" err = new Error("not authorized") - logger.error {err, project_id, user_id, client_id: client.id}, "user is not authorized to join project" + logger.warn {err, project_id, user_id, client_id: client.id}, "user is not authorized to join project" return callback(err) client.join project_id diff --git a/services/real-time/test/unit/coffee/WebsocketControllerTests.coffee b/services/real-time/test/unit/coffee/WebsocketControllerTests.coffee index 965f5d72b2..1ddf4e6359 100644 --- a/services/real-time/test/unit/coffee/WebsocketControllerTests.coffee +++ b/services/real-time/test/unit/coffee/WebsocketControllerTests.coffee @@ -116,7 +116,10 @@ describe 'WebsocketController', -> @callback .calledWith(new Error("not authorized")) .should.equal true - + + it "should not log an error", -> + @logger.error.called.should.equal false + describe "leaveProject", -> beforeEach -> @DocumentUpdaterManager.flushProjectToMongoAndDelete = sinon.stub().callsArg(1) From 283a6066d480a0f4ed655281a8a18c24767650b3 Mon Sep 17 00:00:00 2001 From: Shane Kilkelly Date: Fri, 21 Jun 2019 11:26:28 +0100 Subject: [PATCH 224/491] update logger and metrics --- services/real-time/Makefile | 6 +- services/real-time/buildscript.txt | 2 +- services/real-time/docker-compose.ci.yml | 5 +- services/real-time/docker-compose.yml | 8 +- services/real-time/npm-shrinkwrap.json | 501 ++++++++++------------- services/real-time/package.json | 4 +- 6 files changed, 242 insertions(+), 284 deletions(-) diff --git a/services/real-time/Makefile b/services/real-time/Makefile index fdf938da5a..573659dd90 100644 --- a/services/real-time/Makefile +++ b/services/real-time/Makefile @@ -1,7 +1,7 @@ # This file was auto-generated, do not edit it directly. # Instead run bin/update_build_scripts from # https://github.com/sharelatex/sharelatex-dev-environment -# Version: 1.1.12 +# Version: 1.1.21 BUILD_NUMBER ?= local BRANCH_NAME ?= $(shell git rev-parse --abbrev-ref HEAD) @@ -26,7 +26,9 @@ test: test_unit test_acceptance test_unit: @[ ! -d test/unit ] && echo "real-time has no unit tests" || $(DOCKER_COMPOSE) run --rm test_unit -test_acceptance: test_clean test_acceptance_pre_run # clear the database before each acceptance test run +test_acceptance: test_clean test_acceptance_pre_run test_acceptance_run + +test_acceptance_run: @[ ! -d test/acceptance ] && echo "real-time has no acceptance tests" || $(DOCKER_COMPOSE) run --rm test_acceptance test_clean: diff --git a/services/real-time/buildscript.txt b/services/real-time/buildscript.txt index 5d6ccb471c..1f61f10934 100644 --- a/services/real-time/buildscript.txt +++ b/services/real-time/buildscript.txt @@ -5,4 +5,4 @@ real-time --dependencies=mongo,redis --docker-repos=gcr.io/overleaf-ops --build-target=docker ---script-version=1.1.12 +--script-version=1.1.21 diff --git a/services/real-time/docker-compose.ci.yml b/services/real-time/docker-compose.ci.yml index 36b52f8f8b..d2bcca9ec6 100644 --- a/services/real-time/docker-compose.ci.yml +++ b/services/real-time/docker-compose.ci.yml @@ -1,7 +1,7 @@ # This file was auto-generated, do not edit it directly. # Instead run bin/update_build_scripts from # https://github.com/sharelatex/sharelatex-dev-environment -# Version: 1.1.12 +# Version: 1.1.21 version: "2" @@ -10,6 +10,8 @@ services: image: ci/$PROJECT_NAME:$BRANCH_NAME-$BUILD_NUMBER user: node command: npm run test:unit:_run + environment: + NODE_ENV: test test_acceptance: @@ -21,6 +23,7 @@ services: MONGO_HOST: mongo POSTGRES_HOST: postgres MOCHA_GREP: ${MOCHA_GREP} + NODE_ENV: test depends_on: - mongo - redis diff --git a/services/real-time/docker-compose.yml b/services/real-time/docker-compose.yml index 8bb7857cb6..47b12619da 100644 --- a/services/real-time/docker-compose.yml +++ b/services/real-time/docker-compose.yml @@ -1,18 +1,19 @@ # This file was auto-generated, do not edit it directly. # Instead run bin/update_build_scripts from # https://github.com/sharelatex/sharelatex-dev-environment -# Version: 1.1.12 +# Version: 1.1.21 version: "2" services: test_unit: - build: . + image: node:6.15.1 volumes: - .:/app working_dir: /app environment: MOCHA_GREP: ${MOCHA_GREP} + NODE_ENV: test command: npm run test:unit user: node @@ -27,6 +28,8 @@ services: MONGO_HOST: mongo POSTGRES_HOST: postgres MOCHA_GREP: ${MOCHA_GREP} + LOG_LEVEL: ERROR + NODE_ENV: test user: node depends_on: - mongo @@ -49,3 +52,4 @@ services: mongo: image: mongo:3.4 + diff --git a/services/real-time/npm-shrinkwrap.json b/services/real-time/npm-shrinkwrap.json index 4c0e2a0f85..52f790d729 100644 --- a/services/real-time/npm-shrinkwrap.json +++ b/services/real-time/npm-shrinkwrap.json @@ -3,19 +3,19 @@ "version": "0.1.4", "dependencies": { "@google-cloud/common": { - "version": "0.27.0", - "from": "@google-cloud/common@>=0.27.0 <0.28.0", - "resolved": "https://registry.npmjs.org/@google-cloud/common/-/common-0.27.0.tgz" + "version": "0.32.1", + "from": "@google-cloud/common@>=0.32.0 <0.33.0", + "resolved": "https://registry.npmjs.org/@google-cloud/common/-/common-0.32.1.tgz" }, "@google-cloud/debug-agent": { - "version": "3.0.1", + "version": "3.2.0", "from": "@google-cloud/debug-agent@>=3.0.0 <4.0.0", - "resolved": "https://registry.npmjs.org/@google-cloud/debug-agent/-/debug-agent-3.0.1.tgz", + "resolved": "https://registry.npmjs.org/@google-cloud/debug-agent/-/debug-agent-3.2.0.tgz", "dependencies": { "coffeescript": { - "version": "2.3.2", + "version": "2.4.1", "from": "coffeescript@>=2.0.0 <3.0.0", - "resolved": "https://registry.npmjs.org/coffeescript/-/coffeescript-2.3.2.tgz" + "resolved": "https://registry.npmjs.org/coffeescript/-/coffeescript-2.4.1.tgz" } } }, @@ -29,38 +29,60 @@ "from": "@google-cloud/common@>=0.26.0 <0.27.0", "resolved": "https://registry.npmjs.org/@google-cloud/common/-/common-0.26.2.tgz" }, + "@google-cloud/promisify": { + "version": "0.3.1", + "from": "@google-cloud/promisify@>=0.3.0 <0.4.0", + "resolved": "https://registry.npmjs.org/@google-cloud/promisify/-/promisify-0.3.1.tgz" + }, + "arrify": { + "version": "1.0.1", + "from": "arrify@>=1.0.1 <2.0.0", + "resolved": "https://registry.npmjs.org/arrify/-/arrify-1.0.1.tgz" + }, + "gcp-metadata": { + "version": "0.9.3", + "from": "gcp-metadata@>=0.9.0 <0.10.0", + "resolved": "https://registry.npmjs.org/gcp-metadata/-/gcp-metadata-0.9.3.tgz" + }, + "google-auth-library": { + "version": "2.0.2", + "from": "google-auth-library@>=2.0.0 <3.0.0", + "resolved": "https://registry.npmjs.org/google-auth-library/-/google-auth-library-2.0.2.tgz", + "dependencies": { + "gcp-metadata": { + "version": "0.7.0", + "from": "gcp-metadata@>=0.7.0 <0.8.0", + "resolved": "https://registry.npmjs.org/gcp-metadata/-/gcp-metadata-0.7.0.tgz" + } + } + }, + "semver": { + "version": "5.7.0", + "from": "semver@>=5.5.0 <6.0.0", + "resolved": "https://registry.npmjs.org/semver/-/semver-5.7.0.tgz" + }, "through2": { - "version": "3.0.0", + "version": "3.0.1", "from": "through2@>=3.0.0 <4.0.0", - "resolved": "https://registry.npmjs.org/through2/-/through2-3.0.0.tgz" + "resolved": "https://registry.npmjs.org/through2/-/through2-3.0.1.tgz" } } }, "@google-cloud/projectify": { - "version": "0.3.2", - "from": "@google-cloud/projectify@>=0.3.0 <0.4.0", - "resolved": "https://registry.npmjs.org/@google-cloud/projectify/-/projectify-0.3.2.tgz" + "version": "0.3.3", + "from": "@google-cloud/projectify@>=0.3.3 <0.4.0", + "resolved": "https://registry.npmjs.org/@google-cloud/projectify/-/projectify-0.3.3.tgz" }, "@google-cloud/promisify": { - "version": "0.3.1", - "from": "@google-cloud/promisify@>=0.3.0 <0.4.0", - "resolved": "https://registry.npmjs.org/@google-cloud/promisify/-/promisify-0.3.1.tgz" + "version": "0.4.0", + "from": "@google-cloud/promisify@>=0.4.0 <0.5.0", + "resolved": "https://registry.npmjs.org/@google-cloud/promisify/-/promisify-0.4.0.tgz" }, "@google-cloud/trace-agent": { - "version": "3.5.2", + "version": "3.6.1", "from": "@google-cloud/trace-agent@>=3.2.0 <4.0.0", - "resolved": "https://registry.npmjs.org/@google-cloud/trace-agent/-/trace-agent-3.5.2.tgz", + "resolved": "https://registry.npmjs.org/@google-cloud/trace-agent/-/trace-agent-3.6.1.tgz", "dependencies": { - "@google-cloud/common": { - "version": "0.30.2", - "from": "@google-cloud/common@>=0.30.0 <0.31.0", - "resolved": "https://registry.npmjs.org/@google-cloud/common/-/common-0.30.2.tgz" - }, - "google-auth-library": { - "version": "3.0.1", - "from": "google-auth-library@>=3.0.0 <4.0.0", - "resolved": "https://registry.npmjs.org/google-auth-library/-/google-auth-library-3.0.1.tgz" - }, "uuid": { "version": "3.3.2", "from": "uuid@^3.0.1", @@ -119,14 +141,14 @@ "resolved": "https://registry.npmjs.org/@protobufjs/utf8/-/utf8-1.1.0.tgz" }, "@sindresorhus/is": { - "version": "0.13.0", - "from": "@sindresorhus/is@>=0.13.0 <0.14.0", - "resolved": "https://registry.npmjs.org/@sindresorhus/is/-/is-0.13.0.tgz" + "version": "0.15.0", + "from": "@sindresorhus/is@>=0.15.0 <0.16.0", + "resolved": "https://registry.npmjs.org/@sindresorhus/is/-/is-0.15.0.tgz" }, "@types/caseless": { - "version": "0.12.1", + "version": "0.12.2", "from": "@types/caseless@*", - "resolved": "https://registry.npmjs.org/@types/caseless/-/caseless-0.12.1.tgz" + "resolved": "https://registry.npmjs.org/@types/caseless/-/caseless-0.12.2.tgz" }, "@types/console-log-level": { "version": "1.4.0", @@ -149,9 +171,9 @@ "resolved": "https://registry.npmjs.org/@types/long/-/long-4.0.0.tgz" }, "@types/node": { - "version": "10.12.20", + "version": "12.0.8", "from": "@types/node@*", - "resolved": "https://registry.npmjs.org/@types/node/-/node-10.12.20.tgz" + "resolved": "https://registry.npmjs.org/@types/node/-/node-12.0.8.tgz" }, "@types/request": { "version": "2.48.1", @@ -168,15 +190,20 @@ "from": "@types/tough-cookie@*", "resolved": "https://registry.npmjs.org/@types/tough-cookie/-/tough-cookie-2.3.5.tgz" }, + "abort-controller": { + "version": "3.0.0", + "from": "abort-controller@>=3.0.0 <4.0.0", + "resolved": "https://registry.npmjs.org/abort-controller/-/abort-controller-3.0.0.tgz" + }, "accepts": { "version": "1.3.5", "from": "accepts@>=1.3.5 <1.4.0", "resolved": "https://registry.npmjs.org/accepts/-/accepts-1.3.5.tgz" }, "acorn": { - "version": "5.7.3", - "from": "acorn@>=5.0.3 <6.0.0", - "resolved": "https://registry.npmjs.org/acorn/-/acorn-5.7.3.tgz" + "version": "6.1.1", + "from": "acorn@>=6.0.0 <7.0.0", + "resolved": "https://registry.npmjs.org/acorn/-/acorn-6.1.1.tgz" }, "active-x-obfuscator": { "version": "0.0.1", @@ -184,9 +211,9 @@ "resolved": "https://registry.npmjs.org/active-x-obfuscator/-/active-x-obfuscator-0.0.1.tgz" }, "agent-base": { - "version": "4.2.1", + "version": "4.3.0", "from": "agent-base@>=4.1.0 <5.0.0", - "resolved": "https://registry.npmjs.org/agent-base/-/agent-base-4.2.1.tgz" + "resolved": "https://registry.npmjs.org/agent-base/-/agent-base-4.3.0.tgz" }, "ajv": { "version": "6.7.0", @@ -199,9 +226,9 @@ "resolved": "https://registry.npmjs.org/array-flatten/-/array-flatten-1.1.1.tgz" }, "arrify": { - "version": "1.0.1", - "from": "arrify@>=1.0.1 <2.0.0", - "resolved": "https://registry.npmjs.org/arrify/-/arrify-1.0.1.tgz" + "version": "2.0.1", + "from": "arrify@>=2.0.0 <3.0.0", + "resolved": "https://registry.npmjs.org/arrify/-/arrify-2.0.1.tgz" }, "assert-plus": { "version": "1.0.0", @@ -222,7 +249,14 @@ "async-listener": { "version": "0.6.10", "from": "async-listener@>=0.6.0 <0.7.0", - "resolved": "https://registry.npmjs.org/async-listener/-/async-listener-0.6.10.tgz" + "resolved": "https://registry.npmjs.org/async-listener/-/async-listener-0.6.10.tgz", + "dependencies": { + "semver": { + "version": "5.7.0", + "from": "semver@>=5.3.0 <6.0.0", + "resolved": "https://registry.npmjs.org/semver/-/semver-5.7.0.tgz" + } + } }, "asynckit": { "version": "0.4.0", @@ -240,9 +274,9 @@ "resolved": "https://registry.npmjs.org/aws4/-/aws4-1.8.0.tgz" }, "axios": { - "version": "0.18.0", + "version": "0.18.1", "from": "axios@>=0.18.0 <0.19.0", - "resolved": "http://registry.npmjs.org/axios/-/axios-0.18.0.tgz" + "resolved": "https://registry.npmjs.org/axios/-/axios-0.18.1.tgz" }, "balanced-match": { "version": "1.0.0", @@ -281,9 +315,9 @@ "resolved": "https://registry.npmjs.org/bignumber.js/-/bignumber.js-7.2.1.tgz" }, "bindings": { - "version": "1.4.0", + "version": "1.5.0", "from": "bindings@>=1.2.1 <2.0.0", - "resolved": "https://registry.npmjs.org/bindings/-/bindings-1.4.0.tgz" + "resolved": "https://registry.npmjs.org/bindings/-/bindings-1.5.0.tgz" }, "bintrees": { "version": "1.0.1", @@ -312,9 +346,9 @@ "resolved": "https://registry.npmjs.org/buffer-equal-constant-time/-/buffer-equal-constant-time-1.0.1.tgz" }, "builtin-modules": { - "version": "3.0.0", + "version": "3.1.0", "from": "builtin-modules@>=3.0.0 <4.0.0", - "resolved": "https://registry.npmjs.org/builtin-modules/-/builtin-modules-3.0.0.tgz" + "resolved": "https://registry.npmjs.org/builtin-modules/-/builtin-modules-3.1.0.tgz" }, "bunyan": { "version": "0.22.3", @@ -356,9 +390,9 @@ "resolved": "https://registry.npmjs.org/cluster-key-slot/-/cluster-key-slot-1.0.12.tgz" }, "coffee-script": { - "version": "1.12.4", - "from": "coffee-script@1.12.4", - "resolved": "https://registry.npmjs.org/coffee-script/-/coffee-script-1.12.4.tgz" + "version": "1.6.0", + "from": "coffee-script@1.6.0", + "resolved": "https://registry.npmjs.org/coffee-script/-/coffee-script-1.6.0.tgz" }, "combined-stream": { "version": "1.0.7", @@ -383,9 +417,9 @@ } }, "console-log-level": { - "version": "1.4.0", + "version": "1.4.1", "from": "console-log-level@>=1.4.0 <2.0.0", - "resolved": "https://registry.npmjs.org/console-log-level/-/console-log-level-1.4.0.tgz" + "resolved": "https://registry.npmjs.org/console-log-level/-/console-log-level-1.4.1.tgz" }, "content-disposition": { "version": "0.5.2", @@ -451,9 +485,9 @@ "dev": true }, "delay": { - "version": "4.1.0", + "version": "4.3.0", "from": "delay@>=4.0.1 <5.0.0", - "resolved": "https://registry.npmjs.org/delay/-/delay-4.1.0.tgz" + "resolved": "https://registry.npmjs.org/delay/-/delay-4.3.0.tgz" }, "delayed-stream": { "version": "1.0.0", @@ -483,9 +517,9 @@ "optional": true }, "duplexify": { - "version": "3.6.1", + "version": "3.7.1", "from": "duplexify@>=3.6.0 <4.0.0", - "resolved": "https://registry.npmjs.org/duplexify/-/duplexify-3.6.1.tgz" + "resolved": "https://registry.npmjs.org/duplexify/-/duplexify-3.7.1.tgz" }, "ecc-jsbn": { "version": "0.1.2", @@ -493,9 +527,9 @@ "resolved": "https://registry.npmjs.org/ecc-jsbn/-/ecc-jsbn-0.1.2.tgz" }, "ecdsa-sig-formatter": { - "version": "1.0.10", - "from": "ecdsa-sig-formatter@1.0.10", - "resolved": "https://registry.npmjs.org/ecdsa-sig-formatter/-/ecdsa-sig-formatter-1.0.10.tgz" + "version": "1.0.11", + "from": "ecdsa-sig-formatter@1.0.11", + "resolved": "https://registry.npmjs.org/ecdsa-sig-formatter/-/ecdsa-sig-formatter-1.0.11.tgz" }, "ee-first": { "version": "1.1.1", @@ -523,9 +557,9 @@ "resolved": "https://registry.npmjs.org/ent/-/ent-2.2.0.tgz" }, "es6-promise": { - "version": "4.2.5", + "version": "4.2.8", "from": "es6-promise@>=4.0.3 <5.0.0", - "resolved": "https://registry.npmjs.org/es6-promise/-/es6-promise-4.2.5.tgz" + "resolved": "https://registry.npmjs.org/es6-promise/-/es6-promise-4.2.8.tgz" }, "es6-promisify": { "version": "5.0.0", @@ -548,6 +582,11 @@ "from": "etag@>=1.8.1 <1.9.0", "resolved": "https://registry.npmjs.org/etag/-/etag-1.8.1.tgz" }, + "event-target-shim": { + "version": "5.0.1", + "from": "event-target-shim@>=5.0.0 <6.0.0", + "resolved": "https://registry.npmjs.org/event-target-shim/-/event-target-shim-5.0.1.tgz" + }, "express": { "version": "4.16.3", "from": "express@>=4.10.1 <5.0.0", @@ -656,15 +695,10 @@ "from": "findit2@>=2.2.3 <3.0.0", "resolved": "https://registry.npmjs.org/findit2/-/findit2-2.2.3.tgz" }, - "flexbuffer": { - "version": "0.0.6", - "from": "flexbuffer@0.0.6", - "resolved": "https://registry.npmjs.org/flexbuffer/-/flexbuffer-0.0.6.tgz" - }, "follow-redirects": { - "version": "1.6.1", - "from": "follow-redirects@>=1.3.0 <2.0.0", - "resolved": "https://registry.npmjs.org/follow-redirects/-/follow-redirects-1.6.1.tgz", + "version": "1.5.10", + "from": "follow-redirects@1.5.10", + "resolved": "https://registry.npmjs.org/follow-redirects/-/follow-redirects-1.5.10.tgz", "dependencies": { "debug": { "version": "3.1.0", @@ -700,14 +734,14 @@ "dev": true }, "gaxios": { - "version": "1.2.7", - "from": "gaxios@>=1.0.4 <2.0.0", - "resolved": "https://registry.npmjs.org/gaxios/-/gaxios-1.2.7.tgz" + "version": "1.8.4", + "from": "gaxios@>=1.2.1 <2.0.0", + "resolved": "https://registry.npmjs.org/gaxios/-/gaxios-1.8.4.tgz" }, "gcp-metadata": { - "version": "0.9.3", - "from": "gcp-metadata@>=0.9.0 <0.10.0", - "resolved": "https://registry.npmjs.org/gcp-metadata/-/gcp-metadata-0.9.3.tgz" + "version": "1.0.0", + "from": "gcp-metadata@>=1.0.0 <2.0.0", + "resolved": "https://registry.npmjs.org/gcp-metadata/-/gcp-metadata-1.0.0.tgz" }, "getpass": { "version": "0.1.7", @@ -728,31 +762,31 @@ "optional": true }, "google-auth-library": { - "version": "2.0.2", - "from": "google-auth-library@>=2.0.0 <3.0.0", - "resolved": "https://registry.npmjs.org/google-auth-library/-/google-auth-library-2.0.2.tgz", + "version": "3.1.2", + "from": "google-auth-library@>=3.1.1 <4.0.0", + "resolved": "https://registry.npmjs.org/google-auth-library/-/google-auth-library-3.1.2.tgz", "dependencies": { - "gcp-metadata": { - "version": "0.7.0", - "from": "gcp-metadata@>=0.7.0 <0.8.0", - "resolved": "https://registry.npmjs.org/gcp-metadata/-/gcp-metadata-0.7.0.tgz" + "semver": { + "version": "5.7.0", + "from": "semver@>=5.5.0 <6.0.0", + "resolved": "https://registry.npmjs.org/semver/-/semver-5.7.0.tgz" } } }, "google-p12-pem": { - "version": "1.0.3", + "version": "1.0.4", "from": "google-p12-pem@>=1.0.0 <2.0.0", - "resolved": "https://registry.npmjs.org/google-p12-pem/-/google-p12-pem-1.0.3.tgz" + "resolved": "https://registry.npmjs.org/google-p12-pem/-/google-p12-pem-1.0.4.tgz" }, "gtoken": { - "version": "2.3.2", - "from": "gtoken@>=2.3.0 <3.0.0", - "resolved": "https://registry.npmjs.org/gtoken/-/gtoken-2.3.2.tgz", + "version": "2.3.3", + "from": "gtoken@>=2.3.2 <3.0.0", + "resolved": "https://registry.npmjs.org/gtoken/-/gtoken-2.3.3.tgz", "dependencies": { "mime": { - "version": "2.4.0", + "version": "2.4.4", "from": "mime@>=2.2.0 <3.0.0", - "resolved": "https://registry.npmjs.org/mime/-/mime-2.4.0.tgz" + "resolved": "https://registry.npmjs.org/mime/-/mime-2.4.4.tgz" } } }, @@ -798,9 +832,9 @@ "resolved": "https://registry.npmjs.org/debug/-/debug-3.2.6.tgz" }, "ms": { - "version": "2.1.1", + "version": "2.1.2", "from": "ms@>=2.1.1 <3.0.0", - "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.1.tgz" + "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.2.tgz" } } }, @@ -819,23 +853,6 @@ "from": "inherits@2.0.3", "resolved": "https://registry.npmjs.org/inherits/-/inherits-2.0.3.tgz" }, - "ioredis": { - "version": "4.6.3", - "from": "ioredis@>=4.6.0 <4.7.0", - "resolved": "https://registry.npmjs.org/ioredis/-/ioredis-4.6.3.tgz", - "dependencies": { - "debug": { - "version": "3.2.6", - "from": "debug@>=3.1.0 <4.0.0", - "resolved": "https://registry.npmjs.org/debug/-/debug-3.2.6.tgz" - }, - "ms": { - "version": "2.1.1", - "from": "ms@>=2.1.1 <3.0.0", - "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.1.tgz" - } - } - }, "ipaddr.js": { "version": "1.6.0", "from": "ipaddr.js@1.6.0", @@ -847,9 +864,9 @@ "resolved": "https://registry.npmjs.org/is/-/is-3.3.0.tgz" }, "is-buffer": { - "version": "1.1.6", - "from": "is-buffer@>=1.1.5 <2.0.0", - "resolved": "https://registry.npmjs.org/is-buffer/-/is-buffer-1.1.6.tgz" + "version": "2.0.3", + "from": "is-buffer@>=2.0.2 <3.0.0", + "resolved": "https://registry.npmjs.org/is-buffer/-/is-buffer-2.0.3.tgz" }, "is-typedarray": { "version": "1.0.0", @@ -904,14 +921,14 @@ } }, "jwa": { - "version": "1.2.0", - "from": "jwa@>=1.2.0 <2.0.0", - "resolved": "https://registry.npmjs.org/jwa/-/jwa-1.2.0.tgz" + "version": "1.4.1", + "from": "jwa@>=1.4.1 <2.0.0", + "resolved": "https://registry.npmjs.org/jwa/-/jwa-1.4.1.tgz" }, "jws": { - "version": "3.2.1", + "version": "3.2.2", "from": "jws@>=3.1.5 <4.0.0", - "resolved": "https://registry.npmjs.org/jws/-/jws-3.2.1.tgz" + "resolved": "https://registry.npmjs.org/jws/-/jws-3.2.2.tgz" }, "lodash": { "version": "4.17.11", @@ -934,95 +951,20 @@ "resolved": "https://registry.npmjs.org/lodash.pickby/-/lodash.pickby-4.6.0.tgz" }, "logger-sharelatex": { - "version": "1.6.0", - "from": "logger-sharelatex@1.6.0", - "resolved": "https://registry.npmjs.org/logger-sharelatex/-/logger-sharelatex-1.6.0.tgz", + "version": "1.7.0", + "from": "logger-sharelatex@1.7.0", + "resolved": "https://registry.npmjs.org/logger-sharelatex/-/logger-sharelatex-1.7.0.tgz", "dependencies": { - "assert-plus": { - "version": "1.0.0", - "from": "assert-plus@>=1.0.0 <2.0.0", - "resolved": "https://registry.npmjs.org/assert-plus/-/assert-plus-1.0.0.tgz" - }, - "aws-sign2": { - "version": "0.7.0", - "from": "aws-sign2@>=0.7.0 <0.8.0", - "resolved": "https://registry.npmjs.org/aws-sign2/-/aws-sign2-0.7.0.tgz" - }, "bunyan": { - "version": "1.5.1", - "from": "bunyan@1.5.1", - "resolved": "https://registry.npmjs.org/bunyan/-/bunyan-1.5.1.tgz" - }, - "combined-stream": { - "version": "1.0.7", - "from": "combined-stream@>=1.0.6 <1.1.0", - "resolved": "https://registry.npmjs.org/combined-stream/-/combined-stream-1.0.7.tgz" - }, - "delayed-stream": { - "version": "1.0.0", - "from": "delayed-stream@>=1.0.0 <1.1.0", - "resolved": "https://registry.npmjs.org/delayed-stream/-/delayed-stream-1.0.0.tgz" + "version": "1.8.12", + "from": "bunyan@1.8.12", + "resolved": "https://registry.npmjs.org/bunyan/-/bunyan-1.8.12.tgz" }, "dtrace-provider": { - "version": "0.6.0", - "from": "dtrace-provider@>=0.6.0 <0.7.0", - "resolved": "https://registry.npmjs.org/dtrace-provider/-/dtrace-provider-0.6.0.tgz", + "version": "0.8.7", + "from": "dtrace-provider@>=0.8.0 <0.9.0", + "resolved": "https://registry.npmjs.org/dtrace-provider/-/dtrace-provider-0.8.7.tgz", "optional": true - }, - "forever-agent": { - "version": "0.6.1", - "from": "forever-agent@>=0.6.1 <0.7.0", - "resolved": "https://registry.npmjs.org/forever-agent/-/forever-agent-0.6.1.tgz" - }, - "form-data": { - "version": "2.3.3", - "from": "form-data@>=2.3.2 <2.4.0", - "resolved": "https://registry.npmjs.org/form-data/-/form-data-2.3.3.tgz" - }, - "http-signature": { - "version": "1.2.0", - "from": "http-signature@>=1.2.0 <1.3.0", - "resolved": "https://registry.npmjs.org/http-signature/-/http-signature-1.2.0.tgz" - }, - "mime-db": { - "version": "1.37.0", - "from": "mime-db@>=1.37.0 <1.38.0", - "resolved": "https://registry.npmjs.org/mime-db/-/mime-db-1.37.0.tgz" - }, - "mime-types": { - "version": "2.1.21", - "from": "mime-types@>=2.1.19 <2.2.0", - "resolved": "https://registry.npmjs.org/mime-types/-/mime-types-2.1.21.tgz" - }, - "oauth-sign": { - "version": "0.9.0", - "from": "oauth-sign@>=0.9.0 <0.10.0", - "resolved": "https://registry.npmjs.org/oauth-sign/-/oauth-sign-0.9.0.tgz" - }, - "request": { - "version": "2.88.0", - "from": "request@>=2.88.0 <3.0.0", - "resolved": "https://registry.npmjs.org/request/-/request-2.88.0.tgz" - }, - "safe-buffer": { - "version": "5.1.2", - "from": "safe-buffer@>=5.1.2 <6.0.0", - "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.1.2.tgz" - }, - "tough-cookie": { - "version": "2.4.3", - "from": "tough-cookie@>=2.4.3 <2.5.0", - "resolved": "https://registry.npmjs.org/tough-cookie/-/tough-cookie-2.4.3.tgz" - }, - "tunnel-agent": { - "version": "0.6.0", - "from": "tunnel-agent@>=0.6.0 <0.7.0", - "resolved": "https://registry.npmjs.org/tunnel-agent/-/tunnel-agent-0.6.0.tgz" - }, - "uuid": { - "version": "3.3.2", - "from": "uuid@>=3.3.2 <4.0.0", - "resolved": "https://registry.npmjs.org/uuid/-/uuid-3.3.2.tgz" } } }, @@ -1067,15 +1009,10 @@ "resolved": "https://registry.npmjs.org/methods/-/methods-1.1.2.tgz" }, "metrics-sharelatex": { - "version": "2.1.1", - "from": "metrics-sharelatex@2.1.1", - "resolved": "https://registry.npmjs.org/metrics-sharelatex/-/metrics-sharelatex-2.1.1.tgz", + "version": "2.2.0", + "from": "metrics-sharelatex@2.2.0", + "resolved": "https://registry.npmjs.org/metrics-sharelatex/-/metrics-sharelatex-2.2.0.tgz", "dependencies": { - "coffee-script": { - "version": "1.6.0", - "from": "coffee-script@1.6.0", - "resolved": "https://registry.npmjs.org/coffee-script/-/coffee-script-1.6.0.tgz" - }, "underscore": { "version": "1.6.0", "from": "underscore@>=1.6.0 <1.7.0", @@ -1168,6 +1105,12 @@ "from": "module-details-from-path@>=1.0.3 <2.0.0", "resolved": "https://registry.npmjs.org/module-details-from-path/-/module-details-from-path-1.0.3.tgz" }, + "moment": { + "version": "2.24.0", + "from": "moment@>=2.10.6 <3.0.0", + "resolved": "https://registry.npmjs.org/moment/-/moment-2.24.0.tgz", + "optional": true + }, "ms": { "version": "2.0.0", "from": "ms@2.0.0", @@ -1202,14 +1145,14 @@ "resolved": "https://registry.npmjs.org/negotiator/-/negotiator-0.6.1.tgz" }, "node-fetch": { - "version": "2.3.0", - "from": "node-fetch@>=2.2.0 <3.0.0", - "resolved": "https://registry.npmjs.org/node-fetch/-/node-fetch-2.3.0.tgz" + "version": "2.6.0", + "from": "node-fetch@>=2.3.0 <3.0.0", + "resolved": "https://registry.npmjs.org/node-fetch/-/node-fetch-2.6.0.tgz" }, "node-forge": { - "version": "0.7.6", - "from": "node-forge@>=0.7.4 <0.8.0", - "resolved": "https://registry.npmjs.org/node-forge/-/node-forge-0.7.6.tgz" + "version": "0.8.4", + "from": "node-forge@>=0.8.0 <0.9.0", + "resolved": "https://registry.npmjs.org/node-forge/-/node-forge-0.8.4.tgz" }, "oauth-sign": { "version": "0.9.0", @@ -1237,14 +1180,14 @@ "resolved": "https://registry.npmjs.org/options/-/options-0.0.6.tgz" }, "p-limit": { - "version": "2.1.0", - "from": "p-limit@>=2.0.0 <3.0.0", - "resolved": "https://registry.npmjs.org/p-limit/-/p-limit-2.1.0.tgz" + "version": "2.2.0", + "from": "p-limit@>=2.2.0 <3.0.0", + "resolved": "https://registry.npmjs.org/p-limit/-/p-limit-2.2.0.tgz" }, "p-try": { - "version": "2.0.0", + "version": "2.2.0", "from": "p-try@>=2.0.0 <3.0.0", - "resolved": "https://registry.npmjs.org/p-try/-/p-try-2.0.0.tgz" + "resolved": "https://registry.npmjs.org/p-try/-/p-try-2.2.0.tgz" }, "parse-duration": { "version": "0.1.1", @@ -1252,9 +1195,9 @@ "resolved": "https://registry.npmjs.org/parse-duration/-/parse-duration-0.1.1.tgz" }, "parse-ms": { - "version": "2.0.0", + "version": "2.1.0", "from": "parse-ms@>=2.0.0 <3.0.0", - "resolved": "https://registry.npmjs.org/parse-ms/-/parse-ms-2.0.0.tgz" + "resolved": "https://registry.npmjs.org/parse-ms/-/parse-ms-2.1.0.tgz" }, "parseurl": { "version": "1.3.2", @@ -1302,14 +1245,21 @@ "resolved": "https://registry.npmjs.org/process-nextick-args/-/process-nextick-args-2.0.0.tgz" }, "prom-client": { - "version": "11.2.1", + "version": "11.5.1", "from": "prom-client@>=11.1.3 <12.0.0", - "resolved": "https://registry.npmjs.org/prom-client/-/prom-client-11.2.1.tgz" + "resolved": "https://registry.npmjs.org/prom-client/-/prom-client-11.5.1.tgz" }, "protobufjs": { "version": "6.8.8", "from": "protobufjs@>=6.8.6 <6.9.0", - "resolved": "https://registry.npmjs.org/protobufjs/-/protobufjs-6.8.8.tgz" + "resolved": "https://registry.npmjs.org/protobufjs/-/protobufjs-6.8.8.tgz", + "dependencies": { + "@types/node": { + "version": "10.14.9", + "from": "@types/node@>=10.1.0 <11.0.0", + "resolved": "https://registry.npmjs.org/@types/node/-/node-10.14.9.tgz" + } + } }, "proxy-addr": { "version": "2.0.3", @@ -1347,9 +1297,9 @@ "resolved": "https://registry.npmjs.org/range-parser/-/range-parser-1.2.0.tgz" }, "raven": { - "version": "1.2.1", - "from": "raven@>=1.1.3 <2.0.0", - "resolved": "https://registry.npmjs.org/raven/-/raven-1.2.1.tgz" + "version": "1.1.3", + "from": "raven@1.1.3", + "resolved": "https://registry.npmjs.org/raven/-/raven-1.1.3.tgz" }, "raw-body": { "version": "2.3.3", @@ -1395,46 +1345,43 @@ }, "redis-sharelatex": { "version": "1.0.8", + "from": "redis-sharelatex@1.0.8", "resolved": "https://registry.npmjs.org/redis-sharelatex/-/redis-sharelatex-1.0.8.tgz", - "integrity": "sha512-X88/tG03NKWoy0uMzTrzARvILaFj9ZoKGhECtjf8N7GeCzo90zCeT0cVIJCVHeECogXCxBRf/ABFUBBQKUOCew==", - "requires": { - "async": "^2.5.0", - "coffee-script": "1.8.0", - "ioredis": "~4.9.1", - "redis-sentinel": "0.1.1", - "underscore": "1.7.0" - }, "dependencies": { "async": { "version": "2.6.2", - "resolved": "https://registry.npmjs.org/async/-/async-2.6.2.tgz", - "integrity": "sha512-H1qVYh1MYhEEFLsP97cVKqCGo7KfCyTt6uEWqsTBr9SO84oK9Uwbyd/yCW+6rKJLHksBNUVWZDAjfS+Ccx0Bbg==", - "requires": { - "lodash": "^4.17.11" - } + "from": "async@https://registry.npmjs.org/async/-/async-2.6.2.tgz", + "resolved": "https://registry.npmjs.org/async/-/async-2.6.2.tgz" }, "coffee-script": { "version": "1.8.0", - "resolved": "https://registry.npmjs.org/coffee-script/-/coffee-script-1.8.0.tgz", - "integrity": "sha1-nJ8dK0pSoADe0Vtll5FwNkgmPB0=", - "requires": { - "mkdirp": "~0.3.5" - } + "from": "coffee-script@https://registry.npmjs.org/coffee-script/-/coffee-script-1.8.0.tgz", + "resolved": "https://registry.npmjs.org/coffee-script/-/coffee-script-1.8.0.tgz" }, - "lodash": { - "version": "4.17.11", - "resolved": "https://registry.npmjs.org/lodash/-/lodash-4.17.11.tgz", - "integrity": "sha512-cQKh8igo5QUhZ7lg38DYWAxMvjSAKG0A8wGSVimP07SIUEK2UO+arSRKbRZWtelMtN5V0Hkwh5ryOto/SshYIg==" + "debug": { + "version": "3.2.6", + "from": "debug@>=3.1.0 <4.0.0", + "resolved": "https://registry.npmjs.org/debug/-/debug-3.2.6.tgz" + }, + "ioredis": { + "version": "4.9.5", + "from": "ioredis@>=4.9.1 <4.10.0", + "resolved": "https://registry.npmjs.org/ioredis/-/ioredis-4.9.5.tgz" }, "mkdirp": { "version": "0.3.5", - "resolved": "https://registry.npmjs.org/mkdirp/-/mkdirp-0.3.5.tgz", - "integrity": "sha1-3j5fiWHIjHh+4TaN+EmsRBPsqNc=" + "from": "mkdirp@https://registry.npmjs.org/mkdirp/-/mkdirp-0.3.5.tgz", + "resolved": "https://registry.npmjs.org/mkdirp/-/mkdirp-0.3.5.tgz" }, - "underscore": { - "version": "1.7.0", - "resolved": "https://registry.npmjs.org/underscore/-/underscore-1.7.0.tgz", - "integrity": "sha1-a7rwh3UA02vjTsqlhODbn+8DUgk=" + "ms": { + "version": "2.1.2", + "from": "ms@>=2.1.1 <3.0.0", + "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.2.tgz" + }, + "standard-as-callback": { + "version": "2.0.1", + "from": "standard-as-callback@>=2.0.1 <3.0.0", + "resolved": "https://registry.npmjs.org/standard-as-callback/-/standard-as-callback-2.0.1.tgz" } } }, @@ -1466,9 +1413,21 @@ } }, "require-in-the-middle": { - "version": "3.1.0", - "from": "require-in-the-middle@>=3.0.0 <4.0.0", - "resolved": "https://registry.npmjs.org/require-in-the-middle/-/require-in-the-middle-3.1.0.tgz" + "version": "4.0.0", + "from": "require-in-the-middle@>=4.0.0 <5.0.0", + "resolved": "https://registry.npmjs.org/require-in-the-middle/-/require-in-the-middle-4.0.0.tgz", + "dependencies": { + "debug": { + "version": "4.1.1", + "from": "debug@>=4.1.1 <5.0.0", + "resolved": "https://registry.npmjs.org/debug/-/debug-4.1.1.tgz" + }, + "ms": { + "version": "2.1.2", + "from": "ms@^2.1.1", + "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.2.tgz" + } + } }, "require-like": { "version": "0.1.2", @@ -1477,9 +1436,9 @@ "dev": true }, "resolve": { - "version": "1.10.0", - "from": "resolve@>=1.5.0 <2.0.0", - "resolved": "https://registry.npmjs.org/resolve/-/resolve-1.10.0.tgz" + "version": "1.11.0", + "from": "resolve@>=1.10.0 <2.0.0", + "resolved": "https://registry.npmjs.org/resolve/-/resolve-1.11.0.tgz" }, "retry-axios": { "version": "0.3.2", @@ -1528,9 +1487,9 @@ } }, "semver": { - "version": "5.6.0", - "from": "semver@>=5.5.0 <6.0.0", - "resolved": "https://registry.npmjs.org/semver/-/semver-5.6.0.tgz" + "version": "6.1.1", + "from": "semver@>=6.0.0 <7.0.0", + "resolved": "https://registry.npmjs.org/semver/-/semver-6.1.1.tgz" }, "send": { "version": "0.16.2", @@ -1637,11 +1596,6 @@ "from": "stack-trace@0.0.9", "resolved": "https://registry.npmjs.org/stack-trace/-/stack-trace-0.0.9.tgz" }, - "standard-as-callback": { - "version": "1.0.2", - "from": "standard-as-callback@>=1.0.0 <2.0.0", - "resolved": "https://registry.npmjs.org/standard-as-callback/-/standard-as-callback-1.0.2.tgz" - }, "statsd-parser": { "version": "0.0.4", "from": "statsd-parser@>=0.0.4 <0.1.0", @@ -1662,11 +1616,6 @@ "from": "string_decoder@>=1.1.1 <1.2.0", "resolved": "https://registry.npmjs.org/string_decoder/-/string_decoder-1.1.1.tgz" }, - "symbol-observable": { - "version": "1.2.0", - "from": "symbol-observable@>=1.2.0 <2.0.0", - "resolved": "https://registry.npmjs.org/symbol-observable/-/symbol-observable-1.2.0.tgz" - }, "tdigest": { "version": "0.1.1", "from": "tdigest@>=0.1.1 <0.2.0", diff --git a/services/real-time/package.json b/services/real-time/package.json index 3c8a05591a..6e2f68496e 100644 --- a/services/real-time/package.json +++ b/services/real-time/package.json @@ -28,8 +28,8 @@ "cookie-parser": "^1.3.3", "express": "^4.10.1", "express-session": "^1.9.1", - "logger-sharelatex": "^1.6.0", - "metrics-sharelatex": "^2.1.1", + "logger-sharelatex": "^1.7.0", + "metrics-sharelatex": "^2.2.0", "redis-sharelatex": "^1.0.8", "request": "^2.88.0", "session.socket.io": "^0.1.6", From 922a1a1aba75114e879e39c101819353756e55e1 Mon Sep 17 00:00:00 2001 From: Henry Oswald Date: Tue, 2 Jul 2019 12:04:19 +0100 Subject: [PATCH 225/491] bump redis driver --- services/real-time/package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/services/real-time/package.json b/services/real-time/package.json index 3c8a05591a..898301f1f1 100644 --- a/services/real-time/package.json +++ b/services/real-time/package.json @@ -30,7 +30,7 @@ "express-session": "^1.9.1", "logger-sharelatex": "^1.6.0", "metrics-sharelatex": "^2.1.1", - "redis-sharelatex": "^1.0.8", + "redis-sharelatex": "^1.0.9", "request": "^2.88.0", "session.socket.io": "^0.1.6", "settings-sharelatex": "^1.1.0", From 92f60690f3ddbbbfc09ce549890058d16f70dfe6 Mon Sep 17 00:00:00 2001 From: Henry Oswald Date: Tue, 2 Jul 2019 14:46:58 +0100 Subject: [PATCH 226/491] add redis set --- services/real-time/app.coffee | 1 + 1 file changed, 1 insertion(+) diff --git a/services/real-time/app.coffee b/services/real-time/app.coffee index 6161e63a46..fb8187808b 100644 --- a/services/real-time/app.coffee +++ b/services/real-time/app.coffee @@ -60,6 +60,7 @@ app.get "/debug/events", (req, res, next) -> res.send "debug mode will log next #{Settings.debugEvents} events" rclient = require("redis-sharelatex").createClient(Settings.redis.realtime) +rclient.set "hello", "world" app.get "/health_check/redis", (req, res, next) -> rclient.healthCheck (error) -> if error? From 88b75b8baa294348d2f38ce4da245eb6c1064ceb Mon Sep 17 00:00:00 2001 From: Henry Oswald Date: Tue, 2 Jul 2019 14:56:50 +0100 Subject: [PATCH 227/491] send health check data to all redis backends --- services/real-time/app.coffee | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) diff --git a/services/real-time/app.coffee b/services/real-time/app.coffee index fb8187808b..b8e3e964d6 100644 --- a/services/real-time/app.coffee +++ b/services/real-time/app.coffee @@ -133,16 +133,20 @@ if Settings.forceDrainMsDelay? if Settings.continualPubsubTraffic console.log "continualPubsubTraffic enabled" - pubSubClient = redis.createClient(Settings.redis.documentupdater) + redisClients = [redis.createClient(Settings.redis.documentupdater), redis.createClient(Settings.redis.realtime)] - publishJob = (channel, cb)-> + publishJob = (channel, callback)-> checker = new HealthCheckManager(channel) logger.debug {channel:channel}, "sending pub to keep connection alive" json = JSON.stringify({health_check:true, key: checker.id, date: new Date().toString()}) - pubSubClient.publish channel, json, (err)-> - if err? - logger.err {err, channel}, "error publishing pubsub traffic to redis" - cb(err) + jobs = _.map redisClients, (rclient)-> + return (cb)-> + rclient.publish channel, json, (err)-> + if err? + logger.err {err, channel}, "error publishing pubsub traffic to redis" + return cb(err) + + async.series jobs, callback runPubSubTraffic = -> async.map ["applied-ops", "editor-events"], publishJob, (err)-> From 670ce61da1f0aebc45454384dbfc62c15b059abd Mon Sep 17 00:00:00 2001 From: Henry Oswald Date: Tue, 2 Jul 2019 15:36:17 +0100 Subject: [PATCH 228/491] require underscore --- services/real-time/app.coffee | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/services/real-time/app.coffee b/services/real-time/app.coffee index b8e3e964d6..9a74f57803 100644 --- a/services/real-time/app.coffee +++ b/services/real-time/app.coffee @@ -2,6 +2,7 @@ Metrics = require("metrics-sharelatex") Settings = require "settings-sharelatex" Metrics.initialize(Settings.appName or "real-time") async = require("async") +_ = require "underscore" logger = require "logger-sharelatex" logger.initialize("real-time") @@ -145,7 +146,7 @@ if Settings.continualPubsubTraffic if err? logger.err {err, channel}, "error publishing pubsub traffic to redis" return cb(err) - + async.series jobs, callback runPubSubTraffic = -> From b268285ff6f36aa3e331254647668990497f6f73 Mon Sep 17 00:00:00 2001 From: Henry Oswald Date: Tue, 2 Jul 2019 16:55:07 +0100 Subject: [PATCH 229/491] bump redis-sharelatex (and io redis) to 1.0.9 --- services/real-time/npm-shrinkwrap.json | 54 +++++++++++++------------- services/real-time/package.json | 2 +- 2 files changed, 29 insertions(+), 27 deletions(-) diff --git a/services/real-time/npm-shrinkwrap.json b/services/real-time/npm-shrinkwrap.json index 52f790d729..4f45764207 100644 --- a/services/real-time/npm-shrinkwrap.json +++ b/services/real-time/npm-shrinkwrap.json @@ -853,6 +853,23 @@ "from": "inherits@2.0.3", "resolved": "https://registry.npmjs.org/inherits/-/inherits-2.0.3.tgz" }, + "ioredis": { + "version": "4.11.1", + "from": "ioredis@>=4.11.1 <4.12.0", + "resolved": "https://registry.npmjs.org/ioredis/-/ioredis-4.11.1.tgz", + "dependencies": { + "debug": { + "version": "4.1.1", + "from": "debug@>=4.1.1 <5.0.0", + "resolved": "https://registry.npmjs.org/debug/-/debug-4.1.1.tgz" + }, + "ms": { + "version": "2.1.2", + "from": "ms@>=2.1.1 <3.0.0", + "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.2.tgz" + } + } + }, "ipaddr.js": { "version": "1.6.0", "from": "ipaddr.js@1.6.0", @@ -1317,9 +1334,9 @@ "resolved": "https://registry.npmjs.org/redis/-/redis-0.12.1.tgz" }, "redis-commands": { - "version": "1.4.0", - "from": "redis-commands@1.4.0", - "resolved": "https://registry.npmjs.org/redis-commands/-/redis-commands-1.4.0.tgz" + "version": "1.5.0", + "from": "redis-commands@1.5.0", + "resolved": "https://registry.npmjs.org/redis-commands/-/redis-commands-1.5.0.tgz" }, "redis-errors": { "version": "1.2.0", @@ -1344,9 +1361,9 @@ } }, "redis-sharelatex": { - "version": "1.0.8", - "from": "redis-sharelatex@1.0.8", - "resolved": "https://registry.npmjs.org/redis-sharelatex/-/redis-sharelatex-1.0.8.tgz", + "version": "1.0.9", + "from": "redis-sharelatex@1.0.9", + "resolved": "https://registry.npmjs.org/redis-sharelatex/-/redis-sharelatex-1.0.9.tgz", "dependencies": { "async": { "version": "2.6.2", @@ -1358,30 +1375,10 @@ "from": "coffee-script@https://registry.npmjs.org/coffee-script/-/coffee-script-1.8.0.tgz", "resolved": "https://registry.npmjs.org/coffee-script/-/coffee-script-1.8.0.tgz" }, - "debug": { - "version": "3.2.6", - "from": "debug@>=3.1.0 <4.0.0", - "resolved": "https://registry.npmjs.org/debug/-/debug-3.2.6.tgz" - }, - "ioredis": { - "version": "4.9.5", - "from": "ioredis@>=4.9.1 <4.10.0", - "resolved": "https://registry.npmjs.org/ioredis/-/ioredis-4.9.5.tgz" - }, "mkdirp": { "version": "0.3.5", "from": "mkdirp@https://registry.npmjs.org/mkdirp/-/mkdirp-0.3.5.tgz", "resolved": "https://registry.npmjs.org/mkdirp/-/mkdirp-0.3.5.tgz" - }, - "ms": { - "version": "2.1.2", - "from": "ms@>=2.1.1 <3.0.0", - "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.2.tgz" - }, - "standard-as-callback": { - "version": "2.0.1", - "from": "standard-as-callback@>=2.0.1 <3.0.0", - "resolved": "https://registry.npmjs.org/standard-as-callback/-/standard-as-callback-2.0.1.tgz" } } }, @@ -1596,6 +1593,11 @@ "from": "stack-trace@0.0.9", "resolved": "https://registry.npmjs.org/stack-trace/-/stack-trace-0.0.9.tgz" }, + "standard-as-callback": { + "version": "2.0.1", + "from": "standard-as-callback@>=2.0.1 <3.0.0", + "resolved": "https://registry.npmjs.org/standard-as-callback/-/standard-as-callback-2.0.1.tgz" + }, "statsd-parser": { "version": "0.0.4", "from": "statsd-parser@>=0.0.4 <0.1.0", diff --git a/services/real-time/package.json b/services/real-time/package.json index 6e2f68496e..f4548a98d7 100644 --- a/services/real-time/package.json +++ b/services/real-time/package.json @@ -30,7 +30,7 @@ "express-session": "^1.9.1", "logger-sharelatex": "^1.7.0", "metrics-sharelatex": "^2.2.0", - "redis-sharelatex": "^1.0.8", + "redis-sharelatex": "^1.0.9", "request": "^2.88.0", "session.socket.io": "^0.1.6", "settings-sharelatex": "^1.1.0", From 8042a415ec0b10373d1b53aa1eb71e7bdbbb45bc Mon Sep 17 00:00:00 2001 From: Henry Oswald Date: Wed, 3 Jul 2019 09:55:16 +0100 Subject: [PATCH 230/491] move pubsub traffic over to a pubsub redis connection string --- .../real-time/app/coffee/DocumentUpdaterController.coffee | 2 +- services/real-time/app/coffee/WebsocketLoadBalancer.coffee | 4 ++-- services/real-time/config/settings.defaults.coffee | 6 ++++++ 3 files changed, 9 insertions(+), 3 deletions(-) diff --git a/services/real-time/app/coffee/DocumentUpdaterController.coffee b/services/real-time/app/coffee/DocumentUpdaterController.coffee index 1eb5d21274..490902a837 100644 --- a/services/real-time/app/coffee/DocumentUpdaterController.coffee +++ b/services/real-time/app/coffee/DocumentUpdaterController.coffee @@ -1,7 +1,7 @@ logger = require "logger-sharelatex" settings = require 'settings-sharelatex' redis = require("redis-sharelatex") -rclient = redis.createClient(settings.redis.documentupdater) +rclient = redis.createClient(settings.redis.pubsub) SafeJsonParse = require "./SafeJsonParse" EventLogger = require "./EventLogger" HealthCheckManager = require "./HealthCheckManager" diff --git a/services/real-time/app/coffee/WebsocketLoadBalancer.coffee b/services/real-time/app/coffee/WebsocketLoadBalancer.coffee index ffe6e820ca..e4ed673f8a 100644 --- a/services/real-time/app/coffee/WebsocketLoadBalancer.coffee +++ b/services/real-time/app/coffee/WebsocketLoadBalancer.coffee @@ -2,8 +2,8 @@ Settings = require 'settings-sharelatex' logger = require 'logger-sharelatex' redis = require("redis-sharelatex") SafeJsonParse = require "./SafeJsonParse" -rclientPub = redis.createClient(Settings.redis.realtime) -rclientSub = redis.createClient(Settings.redis.realtime) +rclientPub = redis.createClient(Settings.redis.pubsub) +rclientSub = redis.createClient(Settings.redis.pubsub) EventLogger = require "./EventLogger" HealthCheckManager = require "./HealthCheckManager" diff --git a/services/real-time/config/settings.defaults.coffee b/services/real-time/config/settings.defaults.coffee index 28c51f79be..cc091a83f1 100644 --- a/services/real-time/config/settings.defaults.coffee +++ b/services/real-time/config/settings.defaults.coffee @@ -1,5 +1,11 @@ settings = redis: + + pubsub: + host: process.env['PUBSUB_REDIS_HOST'] or process.env['REDIS_HOST'] or "localhost" + port: process.env['PUBSUB_REDIS_PORT'] or process.env['REDIS_PORT'] or "6379" + password: process.env["PUBSUB_REDIS_PASSWORD"] or process.env["REDIS_PASSWORD"] or "" + realtime: host: process.env['REAL_TIME_REDIS_HOST'] or process.env['REDIS_HOST'] or "localhost" port: process.env['REAL_TIME_REDIS_PORT'] or process.env['REDIS_PORT'] or "6379" From 42e5d2fb6e0210527e8b512ca60b70dd2dcaf7df Mon Sep 17 00:00:00 2001 From: Henry Oswald Date: Mon, 8 Jul 2019 11:17:08 +0100 Subject: [PATCH 231/491] Update app.coffee --- services/real-time/app.coffee | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/services/real-time/app.coffee b/services/real-time/app.coffee index 9a74f57803..0ddfe406e9 100644 --- a/services/real-time/app.coffee +++ b/services/real-time/app.coffee @@ -61,7 +61,7 @@ app.get "/debug/events", (req, res, next) -> res.send "debug mode will log next #{Settings.debugEvents} events" rclient = require("redis-sharelatex").createClient(Settings.redis.realtime) -rclient.set "hello", "world" + app.get "/health_check/redis", (req, res, next) -> rclient.healthCheck (error) -> if error? From 9953c933eeb9e3c436daba05480bae8ff26d1f15 Mon Sep 17 00:00:00 2001 From: Henry Oswald Date: Mon, 8 Jul 2019 11:18:02 +0100 Subject: [PATCH 232/491] Update package.json --- services/real-time/package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/services/real-time/package.json b/services/real-time/package.json index 898301f1f1..3c8a05591a 100644 --- a/services/real-time/package.json +++ b/services/real-time/package.json @@ -30,7 +30,7 @@ "express-session": "^1.9.1", "logger-sharelatex": "^1.6.0", "metrics-sharelatex": "^2.1.1", - "redis-sharelatex": "^1.0.9", + "redis-sharelatex": "^1.0.8", "request": "^2.88.0", "session.socket.io": "^0.1.6", "settings-sharelatex": "^1.1.0", From 1038c5cd0d3735f17ff331137f42dcf90aed027d Mon Sep 17 00:00:00 2001 From: Henry Oswald Date: Mon, 8 Jul 2019 11:53:42 +0100 Subject: [PATCH 233/491] send health check to pubsub channel and use different var name --- services/real-time/app.coffee | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/services/real-time/app.coffee b/services/real-time/app.coffee index 0ddfe406e9..ab45c525b4 100644 --- a/services/real-time/app.coffee +++ b/services/real-time/app.coffee @@ -134,15 +134,15 @@ if Settings.forceDrainMsDelay? if Settings.continualPubsubTraffic console.log "continualPubsubTraffic enabled" - redisClients = [redis.createClient(Settings.redis.documentupdater), redis.createClient(Settings.redis.realtime)] + redisClients = [redis.createClient(Settings.redis.documentupdater), redis.createClient(Settings.redis.pubsub)] publishJob = (channel, callback)-> checker = new HealthCheckManager(channel) logger.debug {channel:channel}, "sending pub to keep connection alive" json = JSON.stringify({health_check:true, key: checker.id, date: new Date().toString()}) - jobs = _.map redisClients, (rclient)-> + jobs = _.map redisClients, (checkRclient)-> return (cb)-> - rclient.publish channel, json, (err)-> + checkRclient.publish channel, json, (err)-> if err? logger.err {err, channel}, "error publishing pubsub traffic to redis" return cb(err) From 520857cf7aed04f3324a84b1801dded1bb5af00d Mon Sep 17 00:00:00 2001 From: Henry Oswald Date: Mon, 8 Jul 2019 12:07:28 +0100 Subject: [PATCH 234/491] simplify redis continual traffic we can't send double health check events to same redis, it causes health check duplicate errors. Commit just sends health check data to pub sub pair and then sends non health check traffic to cluster to keep the connection open --- services/real-time/app.coffee | 14 ++++++-------- 1 file changed, 6 insertions(+), 8 deletions(-) diff --git a/services/real-time/app.coffee b/services/real-time/app.coffee index ab45c525b4..4effc386df 100644 --- a/services/real-time/app.coffee +++ b/services/real-time/app.coffee @@ -134,20 +134,18 @@ if Settings.forceDrainMsDelay? if Settings.continualPubsubTraffic console.log "continualPubsubTraffic enabled" - redisClients = [redis.createClient(Settings.redis.documentupdater), redis.createClient(Settings.redis.pubsub)] + pubsubClient = redis.createClient(Settings.redis.pubsub) + clusterClient = redis.createClient(Settings.redis.websessions) publishJob = (channel, callback)-> checker = new HealthCheckManager(channel) logger.debug {channel:channel}, "sending pub to keep connection alive" json = JSON.stringify({health_check:true, key: checker.id, date: new Date().toString()}) - jobs = _.map redisClients, (checkRclient)-> - return (cb)-> - checkRclient.publish channel, json, (err)-> - if err? - logger.err {err, channel}, "error publishing pubsub traffic to redis" - return cb(err) + pubsubClient.publish channel, json, (err)-> + if err? + logger.err {err, channel}, "error publishing pubsub traffic to redis" + clusterClient.publish "cluster-continual-traffic", {keep: "alive"}, callback - async.series jobs, callback runPubSubTraffic = -> async.map ["applied-ops", "editor-events"], publishJob, (err)-> From b5f9bc422bd9a5aab14a21347d5da2c309d383f6 Mon Sep 17 00:00:00 2001 From: Brian Gough Date: Sat, 29 Jun 2019 12:28:54 +0100 Subject: [PATCH 235/491] support multple redis instances for pubsub --- .../coffee/DocumentUpdaterController.coffee | 13 +++++----- .../app/coffee/WebsocketLoadBalancer.coffee | 18 ++++++------- .../DocumentUpdaterControllerTests.coffee | 25 +++++++++++++------ .../coffee/WebsocketLoadBalancerTests.coffee | 13 +++++----- 4 files changed, 41 insertions(+), 28 deletions(-) diff --git a/services/real-time/app/coffee/DocumentUpdaterController.coffee b/services/real-time/app/coffee/DocumentUpdaterController.coffee index 490902a837..41e3bb571a 100644 --- a/services/real-time/app/coffee/DocumentUpdaterController.coffee +++ b/services/real-time/app/coffee/DocumentUpdaterController.coffee @@ -1,7 +1,6 @@ logger = require "logger-sharelatex" settings = require 'settings-sharelatex' redis = require("redis-sharelatex") -rclient = redis.createClient(settings.redis.pubsub) SafeJsonParse = require "./SafeJsonParse" EventLogger = require "./EventLogger" HealthCheckManager = require "./HealthCheckManager" @@ -12,13 +11,15 @@ MESSAGE_SIZE_LOG_LIMIT = 1024 * 1024 # 1Mb module.exports = DocumentUpdaterController = # DocumentUpdaterController is responsible for updates that come via Redis # Pub/Sub from the document updater. + rclientList: [redis.createClient(settings.redis.pubsub)] listenForUpdatesFromDocumentUpdater: (io) -> - rclient.subscribe "applied-ops" - rclient.on "message", (channel, message) -> - metrics.inc "rclient", 0.001 # global event rate metric - EventLogger.debugEvent(channel, message) if settings.debugEvents > 0 - DocumentUpdaterController._processMessageFromDocumentUpdater(io, channel, message) + for rclient in @rclientList + rclient.subscribe "applied-ops" + rclient.on "message", (channel, message) -> + metrics.inc "rclient", 0.001 # global event rate metric + EventLogger.debugEvent(channel, message) if settings.debugEvents > 0 + DocumentUpdaterController._processMessageFromDocumentUpdater(io, channel, message) _processMessageFromDocumentUpdater: (io, channel, message) -> SafeJsonParse.parse message, (error, message) -> diff --git a/services/real-time/app/coffee/WebsocketLoadBalancer.coffee b/services/real-time/app/coffee/WebsocketLoadBalancer.coffee index e4ed673f8a..5c91cd9fa9 100644 --- a/services/real-time/app/coffee/WebsocketLoadBalancer.coffee +++ b/services/real-time/app/coffee/WebsocketLoadBalancer.coffee @@ -2,14 +2,12 @@ Settings = require 'settings-sharelatex' logger = require 'logger-sharelatex' redis = require("redis-sharelatex") SafeJsonParse = require "./SafeJsonParse" -rclientPub = redis.createClient(Settings.redis.pubsub) -rclientSub = redis.createClient(Settings.redis.pubsub) EventLogger = require "./EventLogger" HealthCheckManager = require "./HealthCheckManager" module.exports = WebsocketLoadBalancer = - rclientPub: rclientPub - rclientSub: rclientSub + rclientPubList: [redis.createClient(Settings.redis.pubsub)] + rclientSubList: [redis.createClient(Settings.redis.pubsub)] emitToRoom: (room_id, message, payload...) -> if !room_id? @@ -20,16 +18,18 @@ module.exports = WebsocketLoadBalancer = message: message payload: payload logger.log {room_id, message, payload, length: data.length}, "emitting to room" - @rclientPub.publish "editor-events", data + for rclientPub in @rclientPubList + rclientPub.publish "editor-events", data emitToAll: (message, payload...) -> @emitToRoom "all", message, payload... listenForEditorEvents: (io) -> - @rclientSub.subscribe "editor-events" - @rclientSub.on "message", (channel, message) -> - EventLogger.debugEvent(channel, message) if Settings.debugEvents > 0 - WebsocketLoadBalancer._processEditorEvent io, channel, message + for rclientSub in @rclientSubList + rclientSub.subscribe "editor-events" + rclientSub.on "message", (channel, message) -> + EventLogger.debugEvent(channel, message) if Settings.debugEvents > 0 + WebsocketLoadBalancer._processEditorEvent io, channel, message _processEditorEvent: (io, channel, message) -> SafeJsonParse.parse message, (error, message) -> diff --git a/services/real-time/test/unit/coffee/DocumentUpdaterControllerTests.coffee b/services/real-time/test/unit/coffee/DocumentUpdaterControllerTests.coffee index 5ed274b36c..24551396d9 100644 --- a/services/real-time/test/unit/coffee/DocumentUpdaterControllerTests.coffee +++ b/services/real-time/test/unit/coffee/DocumentUpdaterControllerTests.coffee @@ -10,6 +10,7 @@ describe "DocumentUpdaterController", -> @doc_id = "doc-id-123" @callback = sinon.stub() @io = { "mock": "socket.io" } + @rclient = [] @EditorUpdatesController = SandboxedModule.require modulePath, requires: "logger-sharelatex": @logger = { error: sinon.stub(), log: sinon.stub(), warn: sinon.stub() } "settings-sharelatex": @settings = @@ -17,9 +18,11 @@ describe "DocumentUpdaterController", -> documentupdater: key_schema: pendingUpdates: ({doc_id}) -> "PendingUpdates:#{doc_id}" - "redis-sharelatex" : - createClient: () => - @rclient = {} + pubsub: null + "redis-sharelatex" : @redis = + createClient: (name) => + @rclient.push(rclientStub = {name:name}) + return rclientStub "./SafeJsonParse": @SafeJsonParse = parse: (data, cb) => cb null, JSON.parse(data) "./EventLogger": @EventLogger = {checkEventOrder: sinon.stub()} @@ -28,15 +31,23 @@ describe "DocumentUpdaterController", -> describe "listenForUpdatesFromDocumentUpdater", -> beforeEach -> - @rclient.subscribe = sinon.stub() - @rclient.on = sinon.stub() + @rclient.length = 0 # clear any existing clients + @EditorUpdatesController.rclientList = [@redis.createClient("first"), @redis.createClient("second")] + @rclient[0].subscribe = sinon.stub() + @rclient[0].on = sinon.stub() + @rclient[1].subscribe = sinon.stub() + @rclient[1].on = sinon.stub() @EditorUpdatesController.listenForUpdatesFromDocumentUpdater() it "should subscribe to the doc-updater stream", -> - @rclient.subscribe.calledWith("applied-ops").should.equal true + @rclient[0].subscribe.calledWith("applied-ops").should.equal true it "should register a callback to handle updates", -> - @rclient.on.calledWith("message").should.equal true + @rclient[0].on.calledWith("message").should.equal true + + it "should subscribe to any additional doc-updater stream", -> + @rclient[1].subscribe.calledWith("applied-ops").should.equal true + @rclient[1].on.calledWith("message").should.equal true describe "_processMessageFromDocumentUpdater", -> describe "with bad JSON", -> diff --git a/services/real-time/test/unit/coffee/WebsocketLoadBalancerTests.coffee b/services/real-time/test/unit/coffee/WebsocketLoadBalancerTests.coffee index e786038ab7..132fdc9d5f 100644 --- a/services/real-time/test/unit/coffee/WebsocketLoadBalancerTests.coffee +++ b/services/real-time/test/unit/coffee/WebsocketLoadBalancerTests.coffee @@ -15,11 +15,12 @@ describe "WebsocketLoadBalancer", -> "./EventLogger": {checkEventOrder: sinon.stub()} "./HealthCheckManager": {check: sinon.stub()} @io = {} - @WebsocketLoadBalancer.rclientPub = publish: sinon.stub() - @WebsocketLoadBalancer.rclientSub = + @WebsocketLoadBalancer.rclientPubList = [{publish: sinon.stub()}] + @WebsocketLoadBalancer.rclientSubList = [{ subscribe: sinon.stub() on: sinon.stub() - + }] + @room_id = "room-id" @message = "message-to-editor" @payload = ["argument one", 42] @@ -29,7 +30,7 @@ describe "WebsocketLoadBalancer", -> @WebsocketLoadBalancer.emitToRoom(@room_id, @message, @payload...) it "should publish the message to redis", -> - @WebsocketLoadBalancer.rclientPub.publish + @WebsocketLoadBalancer.rclientPubList[0].publish .calledWith("editor-events", JSON.stringify( room_id: @room_id, message: @message @@ -53,12 +54,12 @@ describe "WebsocketLoadBalancer", -> @WebsocketLoadBalancer.listenForEditorEvents() it "should subscribe to the editor-events channel", -> - @WebsocketLoadBalancer.rclientSub.subscribe + @WebsocketLoadBalancer.rclientSubList[0].subscribe .calledWith("editor-events") .should.equal true it "should process the events with _processEditorEvent", -> - @WebsocketLoadBalancer.rclientSub.on + @WebsocketLoadBalancer.rclientSubList[0].on .calledWith("message", sinon.match.func) .should.equal true From cb289f2decfe240ef89b080d4c09bc6e926360f8 Mon Sep 17 00:00:00 2001 From: Brian Gough Date: Tue, 9 Jul 2019 11:45:00 +0100 Subject: [PATCH 236/491] make redis client list dynamic based on settings --- .../real-time/app/coffee/DocumentUpdaterController.coffee | 4 ++-- services/real-time/app/coffee/RedisClientManager.coffee | 7 +++++++ services/real-time/app/coffee/WebsocketLoadBalancer.coffee | 6 +++--- 3 files changed, 12 insertions(+), 5 deletions(-) create mode 100644 services/real-time/app/coffee/RedisClientManager.coffee diff --git a/services/real-time/app/coffee/DocumentUpdaterController.coffee b/services/real-time/app/coffee/DocumentUpdaterController.coffee index 41e3bb571a..0c89c7dd06 100644 --- a/services/real-time/app/coffee/DocumentUpdaterController.coffee +++ b/services/real-time/app/coffee/DocumentUpdaterController.coffee @@ -1,6 +1,6 @@ logger = require "logger-sharelatex" settings = require 'settings-sharelatex' -redis = require("redis-sharelatex") +RedisClientManager = require "./RedisClientManager" SafeJsonParse = require "./SafeJsonParse" EventLogger = require "./EventLogger" HealthCheckManager = require "./HealthCheckManager" @@ -11,7 +11,7 @@ MESSAGE_SIZE_LOG_LIMIT = 1024 * 1024 # 1Mb module.exports = DocumentUpdaterController = # DocumentUpdaterController is responsible for updates that come via Redis # Pub/Sub from the document updater. - rclientList: [redis.createClient(settings.redis.pubsub)] + rclientList: RedisClientManager.createClientList(settings.redis.pubsub, settings.redis.unusedpubsub) listenForUpdatesFromDocumentUpdater: (io) -> for rclient in @rclientList diff --git a/services/real-time/app/coffee/RedisClientManager.coffee b/services/real-time/app/coffee/RedisClientManager.coffee new file mode 100644 index 0000000000..32b10668f8 --- /dev/null +++ b/services/real-time/app/coffee/RedisClientManager.coffee @@ -0,0 +1,7 @@ +redis = require("redis-sharelatex") + +modules.export = RedisClientManager = + createClientList: (configs...) + # create a dynamic list of redis clients, excluding any configurations which are not defined + clientList = (redis.createClient(x) for x in configs when x?)) + return clientList \ No newline at end of file diff --git a/services/real-time/app/coffee/WebsocketLoadBalancer.coffee b/services/real-time/app/coffee/WebsocketLoadBalancer.coffee index 5c91cd9fa9..89fdc9a27a 100644 --- a/services/real-time/app/coffee/WebsocketLoadBalancer.coffee +++ b/services/real-time/app/coffee/WebsocketLoadBalancer.coffee @@ -1,13 +1,13 @@ Settings = require 'settings-sharelatex' logger = require 'logger-sharelatex' -redis = require("redis-sharelatex") +RedisClientManager = require "./RedisClientManager" SafeJsonParse = require "./SafeJsonParse" EventLogger = require "./EventLogger" HealthCheckManager = require "./HealthCheckManager" module.exports = WebsocketLoadBalancer = - rclientPubList: [redis.createClient(Settings.redis.pubsub)] - rclientSubList: [redis.createClient(Settings.redis.pubsub)] + rclientPubList: RedisClientManager.createClientList(Settings.redis.pubsub, Settings.redis.unusedpubsub) + rclientSubList: RedisClientManager.createClientList(Settings.redis.pubsub, Settings.redis.unusedpubsub) emitToRoom: (room_id, message, payload...) -> if !room_id? From 999cbd8ee6b2190af0aed828ec58ff7591bdb04e Mon Sep 17 00:00:00 2001 From: Brian Gough Date: Tue, 9 Jul 2019 12:01:58 +0100 Subject: [PATCH 237/491] add a per-client metric --- .../real-time/app/coffee/DocumentUpdaterController.coffee | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/services/real-time/app/coffee/DocumentUpdaterController.coffee b/services/real-time/app/coffee/DocumentUpdaterController.coffee index 0c89c7dd06..7ee05e3030 100644 --- a/services/real-time/app/coffee/DocumentUpdaterController.coffee +++ b/services/real-time/app/coffee/DocumentUpdaterController.coffee @@ -14,12 +14,15 @@ module.exports = DocumentUpdaterController = rclientList: RedisClientManager.createClientList(settings.redis.pubsub, settings.redis.unusedpubsub) listenForUpdatesFromDocumentUpdater: (io) -> - for rclient in @rclientList + for rclient, i in @rclientList rclient.subscribe "applied-ops" rclient.on "message", (channel, message) -> metrics.inc "rclient", 0.001 # global event rate metric EventLogger.debugEvent(channel, message) if settings.debugEvents > 0 DocumentUpdaterController._processMessageFromDocumentUpdater(io, channel, message) + do (i) -> + rclient.on "message", () -> + metrics.inc "rclient-#{i}", 0.001 # per client event rate metric _processMessageFromDocumentUpdater: (io, channel, message) -> SafeJsonParse.parse message, (error, message) -> From 580b10036281581d3323770d8b9939ced949ff27 Mon Sep 17 00:00:00 2001 From: Brian Gough Date: Tue, 9 Jul 2019 12:03:13 +0100 Subject: [PATCH 238/491] only publish to one redis client in WebsocketLoadBalancer but listen to all of them --- services/real-time/app/coffee/WebsocketLoadBalancer.coffee | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/services/real-time/app/coffee/WebsocketLoadBalancer.coffee b/services/real-time/app/coffee/WebsocketLoadBalancer.coffee index 89fdc9a27a..7e2b8ec82a 100644 --- a/services/real-time/app/coffee/WebsocketLoadBalancer.coffee +++ b/services/real-time/app/coffee/WebsocketLoadBalancer.coffee @@ -6,7 +6,7 @@ EventLogger = require "./EventLogger" HealthCheckManager = require "./HealthCheckManager" module.exports = WebsocketLoadBalancer = - rclientPubList: RedisClientManager.createClientList(Settings.redis.pubsub, Settings.redis.unusedpubsub) + rclientPubList: RedisClientManager.createClientList(Settings.redis.pubsub) rclientSubList: RedisClientManager.createClientList(Settings.redis.pubsub, Settings.redis.unusedpubsub) emitToRoom: (room_id, message, payload...) -> From dd54789e2b51f58c3a294513cd55d02a9f10f854 Mon Sep 17 00:00:00 2001 From: Brian Gough Date: Tue, 9 Jul 2019 12:20:59 +0100 Subject: [PATCH 239/491] fix build problems --- services/real-time/app/coffee/RedisClientManager.coffee | 6 +++--- .../test/unit/coffee/WebsocketLoadBalancerTests.coffee | 4 ++-- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/services/real-time/app/coffee/RedisClientManager.coffee b/services/real-time/app/coffee/RedisClientManager.coffee index 32b10668f8..01136f14ac 100644 --- a/services/real-time/app/coffee/RedisClientManager.coffee +++ b/services/real-time/app/coffee/RedisClientManager.coffee @@ -1,7 +1,7 @@ redis = require("redis-sharelatex") -modules.export = RedisClientManager = - createClientList: (configs...) +module.exports = RedisClientManager = + createClientList: (configs...) -> # create a dynamic list of redis clients, excluding any configurations which are not defined - clientList = (redis.createClient(x) for x in configs when x?)) + clientList = (redis.createClient(x) for x in configs when x?) return clientList \ No newline at end of file diff --git a/services/real-time/test/unit/coffee/WebsocketLoadBalancerTests.coffee b/services/real-time/test/unit/coffee/WebsocketLoadBalancerTests.coffee index 132fdc9d5f..14df2df851 100644 --- a/services/real-time/test/unit/coffee/WebsocketLoadBalancerTests.coffee +++ b/services/real-time/test/unit/coffee/WebsocketLoadBalancerTests.coffee @@ -7,8 +7,8 @@ describe "WebsocketLoadBalancer", -> beforeEach -> @rclient = {} @WebsocketLoadBalancer = SandboxedModule.require modulePath, requires: - "redis-sharelatex": - createClient: () => @rclient + "./RedisClientManager": + createClientList: () => [] "logger-sharelatex": @logger = { log: sinon.stub(), error: sinon.stub() } "./SafeJsonParse": @SafeJsonParse = parse: (data, cb) => cb null, JSON.parse(data) From 689a75f39703a523721ff53841c40417df701e22 Mon Sep 17 00:00:00 2001 From: Brian Gough Date: Tue, 9 Jul 2019 14:18:39 +0100 Subject: [PATCH 240/491] add logging for redis clients at start up --- .../app/coffee/DocumentUpdaterController.coffee | 1 + .../real-time/app/coffee/RedisClientManager.coffee | 13 ++++++++++++- .../app/coffee/WebsocketLoadBalancer.coffee | 2 ++ 3 files changed, 15 insertions(+), 1 deletion(-) diff --git a/services/real-time/app/coffee/DocumentUpdaterController.coffee b/services/real-time/app/coffee/DocumentUpdaterController.coffee index 7ee05e3030..b3706f47ba 100644 --- a/services/real-time/app/coffee/DocumentUpdaterController.coffee +++ b/services/real-time/app/coffee/DocumentUpdaterController.coffee @@ -14,6 +14,7 @@ module.exports = DocumentUpdaterController = rclientList: RedisClientManager.createClientList(settings.redis.pubsub, settings.redis.unusedpubsub) listenForUpdatesFromDocumentUpdater: (io) -> + logger.log {rclients: @rclientList.length}, "listening for applied-ops events" for rclient, i in @rclientList rclient.subscribe "applied-ops" rclient.on "message", (channel, message) -> diff --git a/services/real-time/app/coffee/RedisClientManager.coffee b/services/real-time/app/coffee/RedisClientManager.coffee index 01136f14ac..1d573df9b8 100644 --- a/services/real-time/app/coffee/RedisClientManager.coffee +++ b/services/real-time/app/coffee/RedisClientManager.coffee @@ -1,7 +1,18 @@ redis = require("redis-sharelatex") +logger = require 'logger-sharelatex' module.exports = RedisClientManager = createClientList: (configs...) -> # create a dynamic list of redis clients, excluding any configurations which are not defined - clientList = (redis.createClient(x) for x in configs when x?) + clientList = for x in configs when x? + redisType = if x.cluster? + "cluster" + else if x.sentinels? + "sentinel" + else if x.host? + "single" + else + "unknown" + logger.log {redis: redisType}, "creating redis client" + redis.createClient(x) return clientList \ No newline at end of file diff --git a/services/real-time/app/coffee/WebsocketLoadBalancer.coffee b/services/real-time/app/coffee/WebsocketLoadBalancer.coffee index 7e2b8ec82a..10f0c0c646 100644 --- a/services/real-time/app/coffee/WebsocketLoadBalancer.coffee +++ b/services/real-time/app/coffee/WebsocketLoadBalancer.coffee @@ -25,6 +25,8 @@ module.exports = WebsocketLoadBalancer = @emitToRoom "all", message, payload... listenForEditorEvents: (io) -> + logger.log {rclients: @rclientPubList.length}, "publishing editor events" + logger.log {rclients: @rclientSubList.length}, "listening for editor events" for rclientSub in @rclientSubList rclientSub.subscribe "editor-events" rclientSub.on "message", (channel, message) -> From 80f8f2465ec99e0c06b8e9c74bdc558f5f91a6e6 Mon Sep 17 00:00:00 2001 From: Brian Gough Date: Thu, 11 Jul 2019 11:10:33 +0100 Subject: [PATCH 241/491] remove unused pubsub client --- services/real-time/app/coffee/DocumentUpdaterController.coffee | 2 +- services/real-time/app/coffee/WebsocketLoadBalancer.coffee | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/services/real-time/app/coffee/DocumentUpdaterController.coffee b/services/real-time/app/coffee/DocumentUpdaterController.coffee index b3706f47ba..a7d174cace 100644 --- a/services/real-time/app/coffee/DocumentUpdaterController.coffee +++ b/services/real-time/app/coffee/DocumentUpdaterController.coffee @@ -11,7 +11,7 @@ MESSAGE_SIZE_LOG_LIMIT = 1024 * 1024 # 1Mb module.exports = DocumentUpdaterController = # DocumentUpdaterController is responsible for updates that come via Redis # Pub/Sub from the document updater. - rclientList: RedisClientManager.createClientList(settings.redis.pubsub, settings.redis.unusedpubsub) + rclientList: RedisClientManager.createClientList(settings.redis.pubsub) listenForUpdatesFromDocumentUpdater: (io) -> logger.log {rclients: @rclientList.length}, "listening for applied-ops events" diff --git a/services/real-time/app/coffee/WebsocketLoadBalancer.coffee b/services/real-time/app/coffee/WebsocketLoadBalancer.coffee index 10f0c0c646..6de7b7c4e7 100644 --- a/services/real-time/app/coffee/WebsocketLoadBalancer.coffee +++ b/services/real-time/app/coffee/WebsocketLoadBalancer.coffee @@ -7,7 +7,7 @@ HealthCheckManager = require "./HealthCheckManager" module.exports = WebsocketLoadBalancer = rclientPubList: RedisClientManager.createClientList(Settings.redis.pubsub) - rclientSubList: RedisClientManager.createClientList(Settings.redis.pubsub, Settings.redis.unusedpubsub) + rclientSubList: RedisClientManager.createClientList(Settings.redis.pubsub) emitToRoom: (room_id, message, payload...) -> if !room_id? From e632f9f29d3a3ebaf7a8893923456dd22fa1d3e3 Mon Sep 17 00:00:00 2001 From: Brian Gough Date: Thu, 11 Jul 2019 11:11:11 +0100 Subject: [PATCH 242/491] only create per-client metrics when there are multiple redis clients --- .../app/coffee/DocumentUpdaterController.coffee | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/services/real-time/app/coffee/DocumentUpdaterController.coffee b/services/real-time/app/coffee/DocumentUpdaterController.coffee index a7d174cace..ddd5702b54 100644 --- a/services/real-time/app/coffee/DocumentUpdaterController.coffee +++ b/services/real-time/app/coffee/DocumentUpdaterController.coffee @@ -21,9 +21,12 @@ module.exports = DocumentUpdaterController = metrics.inc "rclient", 0.001 # global event rate metric EventLogger.debugEvent(channel, message) if settings.debugEvents > 0 DocumentUpdaterController._processMessageFromDocumentUpdater(io, channel, message) - do (i) -> - rclient.on "message", () -> - metrics.inc "rclient-#{i}", 0.001 # per client event rate metric + # create metrics for each redis instance only when we have multiple redis clients + if @rclientList.length > 1 + for rclient, i in @rclientList + do (i) -> + rclient.on "message", () -> + metrics.inc "rclient-#{i}", 0.001 # per client event rate metric _processMessageFromDocumentUpdater: (io, channel, message) -> SafeJsonParse.parse message, (error, message) -> From 24a4709cffb352bcffe0efef88f53cd2c934adeb Mon Sep 17 00:00:00 2001 From: Brian Gough Date: Mon, 15 Jul 2019 11:14:48 +0100 Subject: [PATCH 243/491] log out of order events now that the rate is lower --- services/real-time/app/coffee/EventLogger.coffee | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/services/real-time/app/coffee/EventLogger.coffee b/services/real-time/app/coffee/EventLogger.coffee index 5e727759d6..332973659b 100644 --- a/services/real-time/app/coffee/EventLogger.coffee +++ b/services/real-time/app/coffee/EventLogger.coffee @@ -35,12 +35,11 @@ module.exports = EventLogger = return # order is ok if (count == previous) metrics.inc "event.#{channel}.duplicate" - if Math.random() < 0.01 - logger.warn {channel:channel, message_id:message_id}, "duplicate event (sampled at 1%)" + logger.warn {channel:channel, message_id:message_id}, "duplicate event" return "duplicate" else metrics.inc "event.#{channel}.out-of-order" - # logger.warn {channel:channel, message_id:message_id, key:key, previous: previous, count:count}, "out of order event" + logger.warn {channel:channel, message_id:message_id, key:key, previous: previous, count:count}, "out of order event" return "out-of-order" _storeEventCount: (key, count) -> From 8a7804f0a76d5271840cdc5cc5b4820f9172c9c2 Mon Sep 17 00:00:00 2001 From: Brian Gough Date: Mon, 15 Jul 2019 13:45:34 +0100 Subject: [PATCH 244/491] make event order check a configuration setting --- services/real-time/app/coffee/DocumentUpdaterController.coffee | 2 +- services/real-time/app/coffee/WebsocketLoadBalancer.coffee | 2 +- services/real-time/config/settings.defaults.coffee | 2 ++ 3 files changed, 4 insertions(+), 2 deletions(-) diff --git a/services/real-time/app/coffee/DocumentUpdaterController.coffee b/services/real-time/app/coffee/DocumentUpdaterController.coffee index ddd5702b54..05b95e5fac 100644 --- a/services/real-time/app/coffee/DocumentUpdaterController.coffee +++ b/services/real-time/app/coffee/DocumentUpdaterController.coffee @@ -34,7 +34,7 @@ module.exports = DocumentUpdaterController = logger.error {err: error, channel}, "error parsing JSON" return if message.op? - if message._id? + if message._id? && settings.checkEventOrder status = EventLogger.checkEventOrder("applied-ops", message._id, message) if status is 'duplicate' return # skip duplicate events diff --git a/services/real-time/app/coffee/WebsocketLoadBalancer.coffee b/services/real-time/app/coffee/WebsocketLoadBalancer.coffee index 6de7b7c4e7..a9d5052410 100644 --- a/services/real-time/app/coffee/WebsocketLoadBalancer.coffee +++ b/services/real-time/app/coffee/WebsocketLoadBalancer.coffee @@ -41,7 +41,7 @@ module.exports = WebsocketLoadBalancer = if message.room_id == "all" io.sockets.emit(message.message, message.payload...) else if message.room_id? - if message._id? + if message._id? && Settings.checkEventOrder status = EventLogger.checkEventOrder("editor-events", message._id, message) if status is "duplicate" return # skip duplicate events diff --git a/services/real-time/config/settings.defaults.coffee b/services/real-time/config/settings.defaults.coffee index cc091a83f1..ceeb65191d 100644 --- a/services/real-time/config/settings.defaults.coffee +++ b/services/real-time/config/settings.defaults.coffee @@ -52,6 +52,8 @@ settings = continualPubsubTraffic: process.env['CONTINUAL_PUBSUB_TRAFFIC'] or false + checkEventOrder: process.env['CHECK_EVENT_ORDER'] or false + sentry: dsn: process.env.SENTRY_DSN From 0c6ba4c1a866b0cf34330cc5d94ee3e34671067d Mon Sep 17 00:00:00 2001 From: Brian Gough Date: Tue, 16 Jul 2019 14:02:52 +0100 Subject: [PATCH 245/491] monkeypatch socket.io to fix frame handler in v0.9.16 --- services/real-time/app.coffee | 3 ++ services/real-time/socket.io.patch.js | 43 +++++++++++++++++++++++++++ 2 files changed, 46 insertions(+) create mode 100644 services/real-time/socket.io.patch.js diff --git a/services/real-time/app.coffee b/services/real-time/app.coffee index 4effc386df..7b53730ce6 100644 --- a/services/real-time/app.coffee +++ b/services/real-time/app.coffee @@ -23,6 +23,9 @@ CookieParser = require("cookie-parser") DrainManager = require("./app/js/DrainManager") HealthCheckManager = require("./app/js/HealthCheckManager") +# work around frame handler bug in socket.io v0.9.16 +require("./socket.io.patch.js") + # Set up socket.io server app = express() Metrics.injectMetricsRoute(app) diff --git a/services/real-time/socket.io.patch.js b/services/real-time/socket.io.patch.js new file mode 100644 index 0000000000..21a9608133 --- /dev/null +++ b/services/real-time/socket.io.patch.js @@ -0,0 +1,43 @@ +var io = require("socket.io"); + +if (io.version === "0.9.16") { + console.log("patching socket.io hybi-16 transport frame prototype"); + var transports = require("socket.io/lib/transports/websocket/hybi-16.js"); + transports.prototype.frame = patchedFrameHandler; + // file hybi-07-12 has the same problem but no browsers are using that protocol now +} + +function patchedFrameHandler(opcode, str) { + var dataBuffer = new Buffer(str), + dataLength = dataBuffer.length, + startOffset = 2, + secondByte = dataLength; + if (dataLength === 65536) { + console.log("fixing invalid frame length in socket.io"); + } + if (dataLength > 65535) { + // original code had > 65536 + startOffset = 10; + secondByte = 127; + } else if (dataLength > 125) { + startOffset = 4; + secondByte = 126; + } + var outputBuffer = new Buffer(dataLength + startOffset); + outputBuffer[0] = opcode; + outputBuffer[1] = secondByte; + dataBuffer.copy(outputBuffer, startOffset); + switch (secondByte) { + case 126: + outputBuffer[2] = dataLength >>> 8; + outputBuffer[3] = dataLength % 256; + break; + case 127: + var l = dataLength; + for (var i = 1; i <= 8; ++i) { + outputBuffer[startOffset - i] = l & 0xff; + l >>>= 8; + } + } + return outputBuffer; +} From 804f4c2bd289e33e9fb28af0adee111848029a83 Mon Sep 17 00:00:00 2001 From: Brian Gough Date: Thu, 18 Jul 2019 11:25:10 +0100 Subject: [PATCH 246/491] listen on separate channels for each project/doc --- .../app/coffee/ChannelManager.coffee | 43 ++++++++++++ .../coffee/DocumentUpdaterController.coffee | 13 +++- .../real-time/app/coffee/RoomManager.coffee | 69 +++++++++++++++++++ .../app/coffee/WebsocketController.coffee | 8 ++- .../app/coffee/WebsocketLoadBalancer.coffee | 14 +++- 5 files changed, 142 insertions(+), 5 deletions(-) create mode 100644 services/real-time/app/coffee/ChannelManager.coffee create mode 100644 services/real-time/app/coffee/RoomManager.coffee diff --git a/services/real-time/app/coffee/ChannelManager.coffee b/services/real-time/app/coffee/ChannelManager.coffee new file mode 100644 index 0000000000..0e6b2fb98e --- /dev/null +++ b/services/real-time/app/coffee/ChannelManager.coffee @@ -0,0 +1,43 @@ +logger = require 'logger-sharelatex' +metrics = require "metrics-sharelatex" + +ClientMap = new Map() # for each redis client, stores a Set of subscribed channels + +# Manage redis pubsub subscriptions for individual projects and docs, ensuring +# that we never subscribe to a channel multiple times. The socket.io side is +# handled by RoomManager. + +module.exports = ChannelManager = + _createNewClientEntry: (rclient) -> + ClientMap.set(rclient, new Set()).get(rclient) + + subscribe: (rclient, baseChannel, id) -> + existingChannelSet = ClientMap.get(rclient) || @_createNewClientEntry(rclient) + channel = "#{baseChannel}:#{id}" + if existingChannelSet.has(channel) + logger.error {channel}, "already subscribed" + else + rclient.subscribe channel + existingChannelSet.add(channel) + logger.log {channel}, "subscribed to new channel" + metrics.inc "subscribe.#{baseChannel}" + + unsubscribe: (rclient, baseChannel, id) -> + existingChannelSet = ClientMap.get(rclient) + channel = "#{baseChannel}:#{id}" + if !existingChannelSet.has(channel) + logger.error {channel}, "not subscribed, cannot unsubscribe" + else + rclient.unsubscribe channel + existingChannelSet.delete(channel) + logger.log {channel}, "unsubscribed from channel" + metrics.inc "unsubscribe.#{baseChannel}" + + publish: (rclient, baseChannel, id, data) -> + if id is 'all' + channel = baseChannel + else + channel = "#{baseChannel}:#{id}" + # we publish on a different client to the subscribe, so we can't + # check for the channel existing here + rclient.publish channel, data \ No newline at end of file diff --git a/services/real-time/app/coffee/DocumentUpdaterController.coffee b/services/real-time/app/coffee/DocumentUpdaterController.coffee index 05b95e5fac..0cc5751d7c 100644 --- a/services/real-time/app/coffee/DocumentUpdaterController.coffee +++ b/services/real-time/app/coffee/DocumentUpdaterController.coffee @@ -4,6 +4,8 @@ RedisClientManager = require "./RedisClientManager" SafeJsonParse = require "./SafeJsonParse" EventLogger = require "./EventLogger" HealthCheckManager = require "./HealthCheckManager" +RoomManager = require "./RoomManager" +ChannelManager = require "./ChannelManager" metrics = require "metrics-sharelatex" MESSAGE_SIZE_LOG_LIMIT = 1024 * 1024 # 1Mb @@ -27,7 +29,16 @@ module.exports = DocumentUpdaterController = do (i) -> rclient.on "message", () -> metrics.inc "rclient-#{i}", 0.001 # per client event rate metric - + for rclient in @rclientList + @handleRoomUpdates(rclient) + + handleRoomUpdates: (rclientSub) -> + roomEvents = RoomManager.eventSource() + roomEvents.on 'doc-active', (doc_id) -> + ChannelManager.subscribe rclientSub, "applied-ops", doc_id + roomEvents.on 'doc-empty', (doc_id) -> + ChannelManager.unsubscribe rclientSub, "applied-ops", doc_id + _processMessageFromDocumentUpdater: (io, channel, message) -> SafeJsonParse.parse message, (error, message) -> if error? diff --git a/services/real-time/app/coffee/RoomManager.coffee b/services/real-time/app/coffee/RoomManager.coffee new file mode 100644 index 0000000000..e9787aa1df --- /dev/null +++ b/services/real-time/app/coffee/RoomManager.coffee @@ -0,0 +1,69 @@ +logger = require 'logger-sharelatex' +{EventEmitter} = require 'events' + +IdMap = new Map() # keep track of whether ids are from projects or docs +RoomEvents = new EventEmitter() + +# Manage socket.io rooms for individual projects and docs +# +# The first time someone joins a project or doc we emit a 'project-active' or +# 'doc-active' event. +# +# When the last person leaves a project or doc, we emit 'project-empty' or +# 'doc-empty' event. +# +# The pubsub side is handled by ChannelManager + +module.exports = RoomManager = + + joinProject: (client, project_id) -> + @_join client, "project", project_id + + joinDoc: (client, doc_id) -> + @_join client, "doc", doc_id + + leaveDoc: (client, doc_id) -> + @_leave client, "doc", doc_id + + leaveProjectAndDocs: (client) -> + # what rooms is this client in? we need to leave them all + for id in @_roomsClientIsIn(client) + entity = IdMap.get(id) + @_leave client, entity, id + + eventSource: () -> + return RoomEvents + + _clientsInRoom: (client, room) -> + nsp = client.namespace.name + name = (nsp + '/') + room; + return (client.manager?.rooms?[name] || []).length + + _roomsClientIsIn: (client) -> + roomList = for fullRoomPath of client.manager.roomClients?[client.id] when fullRoomPath isnt '' + # strip socket.io prefix from room to get original id + [prefix, room] = fullRoomPath.split('/', 2) + room + return roomList + + _join: (client, entity, id) -> + beforeCount = @_clientsInRoom(client, id) + client.join id + afterCount = @_clientsInRoom(client, id) + logger.log {client: client.id, entity, id, beforeCount, afterCount}, "client joined room" + # is this a new room? if so, subscribe + if beforeCount == 0 and afterCount == 1 + logger.log {entity, id}, "room is now active" + RoomEvents.emit "#{entity}-active", id + IdMap.set(id, entity) + + _leave: (client, entity, id) -> + beforeCount = @_clientsInRoom(client, id) + client.leave id + afterCount = @_clientsInRoom(client, id) + logger.log {client: client.id, entity, id, beforeCount, afterCount}, "client left room" + # is the room now empty? if so, unsubscribe + if beforeCount == 1 and afterCount == 0 + logger.log {entity, id}, "room is now empty" + RoomEvents.emit "#{entity}-empty", id + IdMap.delete(id) \ No newline at end of file diff --git a/services/real-time/app/coffee/WebsocketController.coffee b/services/real-time/app/coffee/WebsocketController.coffee index 470b1f2a52..6d8965883f 100644 --- a/services/real-time/app/coffee/WebsocketController.coffee +++ b/services/real-time/app/coffee/WebsocketController.coffee @@ -5,6 +5,7 @@ AuthorizationManager = require "./AuthorizationManager" DocumentUpdaterManager = require "./DocumentUpdaterManager" ConnectedUsersManager = require "./ConnectedUsersManager" WebsocketLoadBalancer = require "./WebsocketLoadBalancer" +RoomManager = require "./RoomManager" Utils = require "./Utils" module.exports = WebsocketController = @@ -25,7 +26,7 @@ module.exports = WebsocketController = logger.warn {err, project_id, user_id, client_id: client.id}, "user is not authorized to join project" return callback(err) - client.join project_id + RoomManager.joinProject(client, project_id) client.set("privilege_level", privilegeLevel) client.set("user_id", user_id) @@ -71,6 +72,7 @@ module.exports = WebsocketController = if err? logger.error {err, project_id, user_id, client_id: client.id}, "error marking client as disconnected" + RoomManager.leaveProjectAndDocs(client) setTimeout () -> remainingClients = io.sockets.clients(project_id) if remainingClients.length == 0 @@ -116,7 +118,7 @@ module.exports = WebsocketController = return callback(err) AuthorizationManager.addAccessToDoc client, doc_id - client.join(doc_id) + RoomManager.joinDoc(client, doc_id) callback null, escapedLines, version, ops, ranges logger.log {user_id, project_id, doc_id, fromVersion, client_id: client.id}, "client joined doc" @@ -124,7 +126,7 @@ module.exports = WebsocketController = metrics.inc "editor.leave-doc" Utils.getClientAttributes client, ["project_id", "user_id"], (error, {project_id, user_id}) -> logger.log {user_id, project_id, doc_id, client_id: client.id}, "client leaving doc" - client.leave doc_id + RoomManager.leaveDoc(client, doc_id) # we could remove permission when user leaves a doc, but because # the connection is per-project, we continue to allow access # after the initial joinDoc since we know they are already authorised. diff --git a/services/real-time/app/coffee/WebsocketLoadBalancer.coffee b/services/real-time/app/coffee/WebsocketLoadBalancer.coffee index a9d5052410..1bb74c6a3e 100644 --- a/services/real-time/app/coffee/WebsocketLoadBalancer.coffee +++ b/services/real-time/app/coffee/WebsocketLoadBalancer.coffee @@ -4,6 +4,8 @@ RedisClientManager = require "./RedisClientManager" SafeJsonParse = require "./SafeJsonParse" EventLogger = require "./EventLogger" HealthCheckManager = require "./HealthCheckManager" +RoomManager = require "./RoomManager" +ChannelManager = require "./ChannelManager" module.exports = WebsocketLoadBalancer = rclientPubList: RedisClientManager.createClientList(Settings.redis.pubsub) @@ -18,8 +20,9 @@ module.exports = WebsocketLoadBalancer = message: message payload: payload logger.log {room_id, message, payload, length: data.length}, "emitting to room" + for rclientPub in @rclientPubList - rclientPub.publish "editor-events", data + ChannelManager.publish rclientPub, "editor-events", room_id, data emitToAll: (message, payload...) -> @emitToRoom "all", message, payload... @@ -32,6 +35,15 @@ module.exports = WebsocketLoadBalancer = rclientSub.on "message", (channel, message) -> EventLogger.debugEvent(channel, message) if Settings.debugEvents > 0 WebsocketLoadBalancer._processEditorEvent io, channel, message + for rclientSub in @rclientSubList + @handleRoomUpdates(rclientSub) + + handleRoomUpdates: (rclientSub) -> + roomEvents = RoomManager.eventSource() + roomEvents.on 'project-active', (project_id) -> + ChannelManager.subscribe rclientSub, "editor-events", project_id + roomEvents.on 'project-empty', (project_id) -> + ChannelManager.unsubscribe rclientSub, "editor-events", project_id _processEditorEvent: (io, channel, message) -> SafeJsonParse.parse message, (error, message) -> From f6f6f549d9de2e88d077bc774be0d44230d8b0df Mon Sep 17 00:00:00 2001 From: Brian Gough Date: Thu, 18 Jul 2019 12:40:14 +0100 Subject: [PATCH 247/491] don't publish on individual channels until explicitly set --- services/real-time/app/coffee/ChannelManager.coffee | 3 ++- services/real-time/config/settings.defaults.coffee | 2 ++ 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/services/real-time/app/coffee/ChannelManager.coffee b/services/real-time/app/coffee/ChannelManager.coffee index 0e6b2fb98e..7ea676267f 100644 --- a/services/real-time/app/coffee/ChannelManager.coffee +++ b/services/real-time/app/coffee/ChannelManager.coffee @@ -1,5 +1,6 @@ logger = require 'logger-sharelatex' metrics = require "metrics-sharelatex" +settings = require "settings-sharelatex" ClientMap = new Map() # for each redis client, stores a Set of subscribed channels @@ -34,7 +35,7 @@ module.exports = ChannelManager = metrics.inc "unsubscribe.#{baseChannel}" publish: (rclient, baseChannel, id, data) -> - if id is 'all' + if id is 'all' or !settings.publishOnIndividualChannels channel = baseChannel else channel = "#{baseChannel}:#{id}" diff --git a/services/real-time/config/settings.defaults.coffee b/services/real-time/config/settings.defaults.coffee index ceeb65191d..a3128e84d1 100644 --- a/services/real-time/config/settings.defaults.coffee +++ b/services/real-time/config/settings.defaults.coffee @@ -54,6 +54,8 @@ settings = checkEventOrder: process.env['CHECK_EVENT_ORDER'] or false + publishOnIndividualChannels: process.env['PUBLISH_ON_INDIVIDUAL_CHANNELS'] or false + sentry: dsn: process.env.SENTRY_DSN From 3bf5dd5d6bc5223ec244d97166992a157bef0254 Mon Sep 17 00:00:00 2001 From: Brian Gough Date: Thu, 18 Jul 2019 14:25:25 +0100 Subject: [PATCH 248/491] clarify errors for subscribe/unsubscribe --- services/real-time/app/coffee/ChannelManager.coffee | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/services/real-time/app/coffee/ChannelManager.coffee b/services/real-time/app/coffee/ChannelManager.coffee index 7ea676267f..4d5c843d10 100644 --- a/services/real-time/app/coffee/ChannelManager.coffee +++ b/services/real-time/app/coffee/ChannelManager.coffee @@ -16,9 +16,9 @@ module.exports = ChannelManager = existingChannelSet = ClientMap.get(rclient) || @_createNewClientEntry(rclient) channel = "#{baseChannel}:#{id}" if existingChannelSet.has(channel) - logger.error {channel}, "already subscribed" + logger.error {channel}, "already subscribed - shouldn't happen" else - rclient.subscribe channel + rclient.subscribe channel # completes in the background existingChannelSet.add(channel) logger.log {channel}, "subscribed to new channel" metrics.inc "subscribe.#{baseChannel}" @@ -27,9 +27,9 @@ module.exports = ChannelManager = existingChannelSet = ClientMap.get(rclient) channel = "#{baseChannel}:#{id}" if !existingChannelSet.has(channel) - logger.error {channel}, "not subscribed, cannot unsubscribe" + logger.error {channel}, "not subscribed - shouldn't happen" else - rclient.unsubscribe channel + rclient.unsubscribe channel # completes in the background existingChannelSet.delete(channel) logger.log {channel}, "unsubscribed from channel" metrics.inc "unsubscribe.#{baseChannel}" From 40353a410fdf59f78caf94c4ee4da44e6e1b2d03 Mon Sep 17 00:00:00 2001 From: Brian Gough Date: Fri, 19 Jul 2019 08:49:57 +0100 Subject: [PATCH 249/491] fix unit tests --- .../DocumentUpdaterControllerTests.coffee | 3 +++ .../coffee/WebsocketControllerTests.coffee | 21 +++++++++++++------ .../coffee/WebsocketLoadBalancerTests.coffee | 7 +++++-- 3 files changed, 23 insertions(+), 8 deletions(-) diff --git a/services/real-time/test/unit/coffee/DocumentUpdaterControllerTests.coffee b/services/real-time/test/unit/coffee/DocumentUpdaterControllerTests.coffee index 24551396d9..b5574c8d16 100644 --- a/services/real-time/test/unit/coffee/DocumentUpdaterControllerTests.coffee +++ b/services/real-time/test/unit/coffee/DocumentUpdaterControllerTests.coffee @@ -11,6 +11,7 @@ describe "DocumentUpdaterController", -> @callback = sinon.stub() @io = { "mock": "socket.io" } @rclient = [] + @RoomEvents = { on: sinon.stub() } @EditorUpdatesController = SandboxedModule.require modulePath, requires: "logger-sharelatex": @logger = { error: sinon.stub(), log: sinon.stub(), warn: sinon.stub() } "settings-sharelatex": @settings = @@ -28,6 +29,8 @@ describe "DocumentUpdaterController", -> "./EventLogger": @EventLogger = {checkEventOrder: sinon.stub()} "./HealthCheckManager": {check: sinon.stub()} "metrics-sharelatex": @metrics = {inc: sinon.stub()} + "./RoomManager" : @RoomManager = { eventSource: sinon.stub().returns @RoomEvents} + "./ChannelManager": @ChannelManager = {} describe "listenForUpdatesFromDocumentUpdater", -> beforeEach -> diff --git a/services/real-time/test/unit/coffee/WebsocketControllerTests.coffee b/services/real-time/test/unit/coffee/WebsocketControllerTests.coffee index 1ddf4e6359..ab442006c2 100644 --- a/services/real-time/test/unit/coffee/WebsocketControllerTests.coffee +++ b/services/real-time/test/unit/coffee/WebsocketControllerTests.coffee @@ -36,7 +36,7 @@ describe 'WebsocketController', -> "metrics-sharelatex": @metrics = inc: sinon.stub() set: sinon.stub() - + "./RoomManager": @RoomManager = {} afterEach -> tk.reset() @@ -54,6 +54,7 @@ describe 'WebsocketController', -> @privilegeLevel = "owner" @ConnectedUsersManager.updateUserPosition = sinon.stub().callsArg(4) @WebApiManager.joinProject = sinon.stub().callsArgWith(2, null, @project, @privilegeLevel) + @RoomManager.joinProject = sinon.stub() @WebsocketController.joinProject @client, @user, @project_id, @callback it "should load the project from web", -> @@ -62,7 +63,7 @@ describe 'WebsocketController', -> .should.equal true it "should join the project room", -> - @client.join.calledWith(@project_id).should.equal true + @RoomManager.joinProject.calledWith(@client, @project_id).should.equal true it "should set the privilege level on the client", -> @client.set.calledWith("privilege_level", @privilegeLevel).should.equal true @@ -125,6 +126,7 @@ describe 'WebsocketController', -> @DocumentUpdaterManager.flushProjectToMongoAndDelete = sinon.stub().callsArg(1) @ConnectedUsersManager.markUserAsDisconnected = sinon.stub().callsArg(2) @WebsocketLoadBalancer.emitToRoom = sinon.stub() + @RoomManager.leaveProjectAndDocs = sinon.stub() @clientsInRoom = [] @io = sockets: @@ -160,6 +162,11 @@ describe 'WebsocketController', -> it "should increment the leave-project metric", -> @metrics.inc.calledWith("editor.leave-project").should.equal true + it "should track the disconnection in RoomManager", -> + @RoomManager.leaveProjectAndDocs + .calledWith(@client) + .should.equal true + describe "when the project is not empty", -> beforeEach -> @clientsInRoom = ["mock-remaining-client"] @@ -230,6 +237,7 @@ describe 'WebsocketController', -> @AuthorizationManager.addAccessToDoc = sinon.stub() @AuthorizationManager.assertClientCanViewProject = sinon.stub().callsArgWith(1, null) @DocumentUpdaterManager.getDocument = sinon.stub().callsArgWith(3, null, @doc_lines, @version, @ranges, @ops) + @RoomManager.joinDoc = sinon.stub() describe "works", -> beforeEach -> @@ -251,8 +259,8 @@ describe 'WebsocketController', -> .should.equal true it "should join the client to room for the doc_id", -> - @client.join - .calledWith(@doc_id) + @RoomManager.joinDoc + .calledWith(@client, @doc_id) .should.equal true it "should call the callback with the lines, version, ranges and ops", -> @@ -330,11 +338,12 @@ describe 'WebsocketController', -> beforeEach -> @doc_id = "doc-id-123" @client.params.project_id = @project_id + @RoomManager.leaveDoc = sinon.stub() @WebsocketController.leaveDoc @client, @doc_id, @callback it "should remove the client from the doc_id room", -> - @client.leave - .calledWith(@doc_id).should.equal true + @RoomManager.leaveDoc + .calledWith(@client, @doc_id).should.equal true it "should call the callback", -> @callback.called.should.equal true diff --git a/services/real-time/test/unit/coffee/WebsocketLoadBalancerTests.coffee b/services/real-time/test/unit/coffee/WebsocketLoadBalancerTests.coffee index 14df2df851..c4f4519790 100644 --- a/services/real-time/test/unit/coffee/WebsocketLoadBalancerTests.coffee +++ b/services/real-time/test/unit/coffee/WebsocketLoadBalancerTests.coffee @@ -6,6 +6,7 @@ modulePath = require('path').join __dirname, '../../../app/js/WebsocketLoadBalan describe "WebsocketLoadBalancer", -> beforeEach -> @rclient = {} + @RoomEvents = {on: sinon.stub()} @WebsocketLoadBalancer = SandboxedModule.require modulePath, requires: "./RedisClientManager": createClientList: () => [] @@ -14,6 +15,8 @@ describe "WebsocketLoadBalancer", -> parse: (data, cb) => cb null, JSON.parse(data) "./EventLogger": {checkEventOrder: sinon.stub()} "./HealthCheckManager": {check: sinon.stub()} + "./RoomManager" : @RoomManager = {eventSource: sinon.stub().returns @RoomEvents} + "./ChannelManager": @ChannelManager = {publish: sinon.stub()} @io = {} @WebsocketLoadBalancer.rclientPubList = [{publish: sinon.stub()}] @WebsocketLoadBalancer.rclientSubList = [{ @@ -30,8 +33,8 @@ describe "WebsocketLoadBalancer", -> @WebsocketLoadBalancer.emitToRoom(@room_id, @message, @payload...) it "should publish the message to redis", -> - @WebsocketLoadBalancer.rclientPubList[0].publish - .calledWith("editor-events", JSON.stringify( + @ChannelManager.publish + .calledWith(@WebsocketLoadBalancer.rclientPubList[0], "editor-events", @room_id, JSON.stringify( room_id: @room_id, message: @message payload: @payload From 616014e05d0f3b6f304253bf3ac5fa8a4b4939a8 Mon Sep 17 00:00:00 2001 From: Brian Gough Date: Fri, 19 Jul 2019 08:50:43 +0100 Subject: [PATCH 250/491] add comment about automatically leaving rooms --- services/real-time/app/coffee/RoomManager.coffee | 2 ++ 1 file changed, 2 insertions(+) diff --git a/services/real-time/app/coffee/RoomManager.coffee b/services/real-time/app/coffee/RoomManager.coffee index e9787aa1df..be44edd002 100644 --- a/services/real-time/app/coffee/RoomManager.coffee +++ b/services/real-time/app/coffee/RoomManager.coffee @@ -27,6 +27,8 @@ module.exports = RoomManager = leaveProjectAndDocs: (client) -> # what rooms is this client in? we need to leave them all + # FIXME: socket.io will cause us to leave the rooms, so we only need + # to manage our channel subscriptions for id in @_roomsClientIsIn(client) entity = IdMap.get(id) @_leave client, entity, id From a538d104885ded31dde3b23d6533de93f1f6734b Mon Sep 17 00:00:00 2001 From: Brian Gough Date: Fri, 19 Jul 2019 08:56:38 +0100 Subject: [PATCH 251/491] extend comment re disconnection --- services/real-time/app/coffee/RoomManager.coffee | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/services/real-time/app/coffee/RoomManager.coffee b/services/real-time/app/coffee/RoomManager.coffee index be44edd002..d3a3bea5e8 100644 --- a/services/real-time/app/coffee/RoomManager.coffee +++ b/services/real-time/app/coffee/RoomManager.coffee @@ -26,9 +26,11 @@ module.exports = RoomManager = @_leave client, "doc", doc_id leaveProjectAndDocs: (client) -> - # what rooms is this client in? we need to leave them all - # FIXME: socket.io will cause us to leave the rooms, so we only need - # to manage our channel subscriptions + # what rooms is this client in? we need to leave them all. socket.io + # will cause us to leave the rooms, so we only need to manage our + # channel subscriptions... but it will be safer if we leave them + # explicitly, and then socket.io will just regard this as a client that + # has not joined any rooms and do a final disconnection. for id in @_roomsClientIsIn(client) entity = IdMap.get(id) @_leave client, entity, id @@ -36,6 +38,9 @@ module.exports = RoomManager = eventSource: () -> return RoomEvents + # internal functions below, these access socket.io rooms data directly and + # will need updating for socket.io v2 + _clientsInRoom: (client, room) -> nsp = client.namespace.name name = (nsp + '/') + room; From 9f7df5f10c5b1692f8b7b2429e5e605fb0950fd9 Mon Sep 17 00:00:00 2001 From: Brian Gough Date: Fri, 19 Jul 2019 11:58:40 +0100 Subject: [PATCH 252/491] wip unit tests --- .../test/unit/coffee/ChannelManager.coffee | 33 +++++++++++++++++ .../test/unit/coffee/RoomManagerTests.coffee | 37 +++++++++++++++++++ 2 files changed, 70 insertions(+) create mode 100644 services/real-time/test/unit/coffee/ChannelManager.coffee create mode 100644 services/real-time/test/unit/coffee/RoomManagerTests.coffee diff --git a/services/real-time/test/unit/coffee/ChannelManager.coffee b/services/real-time/test/unit/coffee/ChannelManager.coffee new file mode 100644 index 0000000000..314e9914e1 --- /dev/null +++ b/services/real-time/test/unit/coffee/ChannelManager.coffee @@ -0,0 +1,33 @@ +chai = require('chai') +should = chai.should() +sinon = require("sinon") +modulePath = "../../../app/js/ChannelManager.js" +SandboxedModule = require('sandboxed-module') + +describe 'ChannelManager', -> + beforeEach -> + @project_id = "project-id-123" + @user_id = "user-id-123" + @user = {_id: @user_id} + @callback = sinon.stub() + @ChannelManager = SandboxedModule.require modulePath, requires: + "settings-sharelatex": @settings = {} + "logger-sharelatex": @logger = { log: sinon.stub(), error: sinon.stub() } + + describe "subscribe", -> + + describe "when the project room is empty", -> + + describe "when there are other clients in the project room", -> + + describe "unsubscribe", -> + + describe "when the doc room is empty", -> + + describe "when there are other clients in the doc room", -> + + describe "publish", -> + + describe "when the channel is 'all'", -> + + describe "when the channel has an specific id", -> diff --git a/services/real-time/test/unit/coffee/RoomManagerTests.coffee b/services/real-time/test/unit/coffee/RoomManagerTests.coffee new file mode 100644 index 0000000000..75d241cf6b --- /dev/null +++ b/services/real-time/test/unit/coffee/RoomManagerTests.coffee @@ -0,0 +1,37 @@ +chai = require('chai') +should = chai.should() +sinon = require("sinon") +modulePath = "../../../app/js/RoomManager.js" +SandboxedModule = require('sandboxed-module') + +describe 'RoomManager', -> + beforeEach -> + @project_id = "project-id-123" + @user_id = "user-id-123" + @user = {_id: @user_id} + @callback = sinon.stub() + @RoomManager = SandboxedModule.require modulePath, requires: + "settings-sharelatex": @settings = {} + "logger-sharelatex": @logger = { log: sinon.stub(), error: sinon.stub() } + + describe "joinProject", -> + + describe "when the project room is empty", -> + + describe "when there are other clients in the project room", -> + + describe "joinDoc", -> + + describe "when the doc room is empty", -> + + describe "when there are other clients in the doc room", -> + + describe "leaveDoc", -> + + describe "when doc room will be empty after this client has left", -> + + describe "when there are other clients in the doc room", -> + + describe "leaveProjectAndDocs", -> + + describe "when the client is connected to multiple docs", -> \ No newline at end of file From 8c7b73480f7a08c1ea9dc467d697189b43bb2378 Mon Sep 17 00:00:00 2001 From: Brian Gough Date: Mon, 22 Jul 2019 11:23:02 +0100 Subject: [PATCH 253/491] upgrade sinon to 1.17.7 for onCall support --- services/real-time/npm-shrinkwrap.json | 134 ++++++++++++++++++++++--- services/real-time/package.json | 4 +- 2 files changed, 121 insertions(+), 17 deletions(-) diff --git a/services/real-time/npm-shrinkwrap.json b/services/real-time/npm-shrinkwrap.json index 4f45764207..a146f50e1f 100644 --- a/services/real-time/npm-shrinkwrap.json +++ b/services/real-time/npm-shrinkwrap.json @@ -356,18 +356,6 @@ "resolved": "https://registry.npmjs.org/bunyan/-/bunyan-0.22.3.tgz", "dev": true }, - "buster-core": { - "version": "0.6.4", - "from": "buster-core@0.6.4", - "resolved": "https://registry.npmjs.org/buster-core/-/buster-core-0.6.4.tgz", - "dev": true - }, - "buster-format": { - "version": "0.5.6", - "from": "buster-format@>=0.5.0 <0.6.0", - "resolved": "https://registry.npmjs.org/buster-format/-/buster-format-0.5.6.tgz", - "dev": true - }, "bytes": { "version": "3.0.0", "from": "bytes@3.0.0", @@ -484,6 +472,12 @@ "resolved": "https://registry.npmjs.org/deep-eql/-/deep-eql-0.1.3.tgz", "dev": true }, + "define-properties": { + "version": "1.1.3", + "from": "define-properties@>=1.1.3 <2.0.0", + "resolved": "https://registry.npmjs.org/define-properties/-/define-properties-1.1.3.tgz", + "dev": true + }, "delay": { "version": "4.3.0", "from": "delay@>=4.0.1 <5.0.0", @@ -556,6 +550,18 @@ "from": "ent@>=2.2.0 <3.0.0", "resolved": "https://registry.npmjs.org/ent/-/ent-2.2.0.tgz" }, + "es-abstract": { + "version": "1.13.0", + "from": "es-abstract@>=1.12.0 <2.0.0", + "resolved": "https://registry.npmjs.org/es-abstract/-/es-abstract-1.13.0.tgz", + "dev": true + }, + "es-to-primitive": { + "version": "1.2.0", + "from": "es-to-primitive@>=1.2.0 <2.0.0", + "resolved": "https://registry.npmjs.org/es-to-primitive/-/es-to-primitive-1.2.0.tgz", + "dev": true + }, "es6-promise": { "version": "4.2.8", "from": "es6-promise@>=4.0.3 <5.0.0", @@ -717,6 +723,12 @@ "from": "form-data@>=2.3.2 <2.4.0", "resolved": "https://registry.npmjs.org/form-data/-/form-data-2.3.3.tgz" }, + "formatio": { + "version": "1.1.1", + "from": "formatio@1.1.1", + "resolved": "https://registry.npmjs.org/formatio/-/formatio-1.1.1.tgz", + "dev": true + }, "forwarded": { "version": "0.1.2", "from": "forwarded@>=0.1.2 <0.2.0", @@ -733,6 +745,12 @@ "resolved": "https://registry.npmjs.org/fs.realpath/-/fs.realpath-1.0.0.tgz", "dev": true }, + "function-bind": { + "version": "1.1.1", + "from": "function-bind@>=1.1.1 <2.0.0", + "resolved": "https://registry.npmjs.org/function-bind/-/function-bind-1.1.1.tgz", + "dev": true + }, "gaxios": { "version": "1.8.4", "from": "gaxios@>=1.2.1 <2.0.0", @@ -800,6 +818,18 @@ "from": "har-validator@>=5.1.0 <5.2.0", "resolved": "https://registry.npmjs.org/har-validator/-/har-validator-5.1.3.tgz" }, + "has": { + "version": "1.0.3", + "from": "has@>=1.0.3 <2.0.0", + "resolved": "https://registry.npmjs.org/has/-/has-1.0.3.tgz", + "dev": true + }, + "has-symbols": { + "version": "1.0.0", + "from": "has-symbols@>=1.0.0 <2.0.0", + "resolved": "https://registry.npmjs.org/has-symbols/-/has-symbols-1.0.0.tgz", + "dev": true + }, "he": { "version": "1.1.1", "from": "he@1.1.1", @@ -880,11 +910,47 @@ "from": "is@>=3.2.0 <4.0.0", "resolved": "https://registry.npmjs.org/is/-/is-3.3.0.tgz" }, + "is-arguments": { + "version": "1.0.4", + "from": "is-arguments@>=1.0.4 <2.0.0", + "resolved": "https://registry.npmjs.org/is-arguments/-/is-arguments-1.0.4.tgz", + "dev": true + }, "is-buffer": { "version": "2.0.3", "from": "is-buffer@>=2.0.2 <3.0.0", "resolved": "https://registry.npmjs.org/is-buffer/-/is-buffer-2.0.3.tgz" }, + "is-callable": { + "version": "1.1.4", + "from": "is-callable@>=1.1.4 <2.0.0", + "resolved": "https://registry.npmjs.org/is-callable/-/is-callable-1.1.4.tgz", + "dev": true + }, + "is-date-object": { + "version": "1.0.1", + "from": "is-date-object@>=1.0.1 <2.0.0", + "resolved": "https://registry.npmjs.org/is-date-object/-/is-date-object-1.0.1.tgz", + "dev": true + }, + "is-generator-function": { + "version": "1.0.7", + "from": "is-generator-function@>=1.0.7 <2.0.0", + "resolved": "https://registry.npmjs.org/is-generator-function/-/is-generator-function-1.0.7.tgz", + "dev": true + }, + "is-regex": { + "version": "1.0.4", + "from": "is-regex@>=1.0.4 <2.0.0", + "resolved": "https://registry.npmjs.org/is-regex/-/is-regex-1.0.4.tgz", + "dev": true + }, + "is-symbol": { + "version": "1.0.2", + "from": "is-symbol@>=1.0.2 <2.0.0", + "resolved": "https://registry.npmjs.org/is-symbol/-/is-symbol-1.0.2.tgz", + "dev": true + }, "is-typedarray": { "version": "1.0.0", "from": "is-typedarray@>=1.0.0 <1.1.0", @@ -985,6 +1051,12 @@ } } }, + "lolex": { + "version": "1.3.2", + "from": "lolex@1.3.2", + "resolved": "https://registry.npmjs.org/lolex/-/lolex-1.3.2.tgz", + "dev": true + }, "long": { "version": "4.0.0", "from": "long@>=4.0.0 <5.0.0", @@ -1176,6 +1248,18 @@ "from": "oauth-sign@>=0.9.0 <0.10.0", "resolved": "https://registry.npmjs.org/oauth-sign/-/oauth-sign-0.9.0.tgz" }, + "object-keys": { + "version": "1.1.1", + "from": "object-keys@>=1.0.12 <2.0.0", + "resolved": "https://registry.npmjs.org/object-keys/-/object-keys-1.1.1.tgz", + "dev": true + }, + "object.entries": { + "version": "1.1.0", + "from": "object.entries@>=1.1.0 <2.0.0", + "resolved": "https://registry.npmjs.org/object.entries/-/object.entries-1.1.0.tgz", + "dev": true + }, "on-finished": { "version": "2.3.0", "from": "on-finished@>=2.3.0 <2.4.0", @@ -1469,6 +1553,12 @@ "from": "safer-buffer@>=2.1.2 <3.0.0", "resolved": "https://registry.npmjs.org/safer-buffer/-/safer-buffer-2.1.2.tgz" }, + "samsam": { + "version": "1.1.2", + "from": "samsam@1.1.2", + "resolved": "https://registry.npmjs.org/samsam/-/samsam-1.1.2.tgz", + "dev": true + }, "sandboxed-module": { "version": "0.3.0", "from": "sandboxed-module@>=0.3.0 <0.4.0", @@ -1533,9 +1623,9 @@ "resolved": "https://registry.npmjs.org/shimmer/-/shimmer-1.2.1.tgz" }, "sinon": { - "version": "1.5.2", - "from": "sinon@>=1.5.2 <1.6.0", - "resolved": "https://registry.npmjs.org/sinon/-/sinon-1.5.2.tgz", + "version": "1.17.7", + "from": "sinon@1.17.7", + "resolved": "https://registry.npmjs.org/sinon/-/sinon-1.17.7.tgz", "dev": true }, "socket.io": { @@ -1715,6 +1805,20 @@ } } }, + "util": { + "version": "0.12.1", + "from": "util@>=0.10.3 <1.0.0", + "resolved": "https://registry.npmjs.org/util/-/util-0.12.1.tgz", + "dev": true, + "dependencies": { + "safe-buffer": { + "version": "5.2.0", + "from": "safe-buffer@>=5.1.2 <6.0.0", + "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.2.0.tgz", + "dev": true + } + } + }, "util-deprecate": { "version": "1.0.2", "from": "util-deprecate@>=1.0.1 <1.1.0", diff --git a/services/real-time/package.json b/services/real-time/package.json index f4548a98d7..dd2d6f0198 100644 --- a/services/real-time/package.json +++ b/services/real-time/package.json @@ -42,9 +42,9 @@ "chai": "~1.9.1", "cookie-signature": "^1.0.5", "sandboxed-module": "~0.3.0", - "sinon": "~1.5.2", + "sinon": "^1.5.2", "mocha": "^4.0.1", "uid-safe": "^1.0.1", "timekeeper": "0.0.4" } -} +} \ No newline at end of file From 92e691018078f0c9535ab5097369c6ee446583bd Mon Sep 17 00:00:00 2001 From: Brian Gough Date: Mon, 22 Jul 2019 11:23:33 +0100 Subject: [PATCH 254/491] cleanup --- .../app/coffee/ChannelManager.coffee | 10 ++-- .../real-time/app/coffee/RoomManager.coffee | 55 ++++++++++--------- 2 files changed, 35 insertions(+), 30 deletions(-) diff --git a/services/real-time/app/coffee/ChannelManager.coffee b/services/real-time/app/coffee/ChannelManager.coffee index 4d5c843d10..1eca7a9143 100644 --- a/services/real-time/app/coffee/ChannelManager.coffee +++ b/services/real-time/app/coffee/ChannelManager.coffee @@ -9,11 +9,13 @@ ClientMap = new Map() # for each redis client, stores a Set of subscribed channe # handled by RoomManager. module.exports = ChannelManager = - _createNewClientEntry: (rclient) -> - ClientMap.set(rclient, new Set()).get(rclient) + getClientMapEntry: (rclient) -> + # return the rclient channel set if it exists, otherwise create and + # return an empty set for the client. + ClientMap.get(rclient) || ClientMap.set(rclient, new Set()).get(rclient) subscribe: (rclient, baseChannel, id) -> - existingChannelSet = ClientMap.get(rclient) || @_createNewClientEntry(rclient) + existingChannelSet = @getClientMapEntry(rclient) channel = "#{baseChannel}:#{id}" if existingChannelSet.has(channel) logger.error {channel}, "already subscribed - shouldn't happen" @@ -24,7 +26,7 @@ module.exports = ChannelManager = metrics.inc "subscribe.#{baseChannel}" unsubscribe: (rclient, baseChannel, id) -> - existingChannelSet = ClientMap.get(rclient) + existingChannelSet = @getClientMapEntry(rclient) channel = "#{baseChannel}:#{id}" if !existingChannelSet.has(channel) logger.error {channel}, "not subscribed - shouldn't happen" diff --git a/services/real-time/app/coffee/RoomManager.coffee b/services/real-time/app/coffee/RoomManager.coffee index d3a3bea5e8..225dd37f6d 100644 --- a/services/real-time/app/coffee/RoomManager.coffee +++ b/services/real-time/app/coffee/RoomManager.coffee @@ -17,13 +17,13 @@ RoomEvents = new EventEmitter() module.exports = RoomManager = joinProject: (client, project_id) -> - @_join client, "project", project_id + @joinEntity client, "project", project_id joinDoc: (client, doc_id) -> - @_join client, "doc", doc_id + @joinEntity client, "doc", doc_id leaveDoc: (client, doc_id) -> - @_leave client, "doc", doc_id + @leaveEntity client, "doc", doc_id leaveProjectAndDocs: (client) -> # what rooms is this client in? we need to leave them all. socket.io @@ -33,11 +33,36 @@ module.exports = RoomManager = # has not joined any rooms and do a final disconnection. for id in @_roomsClientIsIn(client) entity = IdMap.get(id) - @_leave client, entity, id + @leaveEntity client, entity, id eventSource: () -> return RoomEvents + joinEntity: (client, entity, id) -> + beforeCount = @_clientsInRoom(client, id) + client.join id + afterCount = @_clientsInRoom(client, id) + logger.log {client: client.id, entity, id, beforeCount, afterCount}, "client joined room" + # is this a new room? if so, subscribe + if beforeCount == 0 and afterCount == 1 + logger.log {entity, id}, "room is now active" + RoomEvents.emit "#{entity}-active", id + IdMap.set(id, entity) + + leaveEntity: (client, entity, id) -> + beforeCount = @_clientsInRoom(client, id) + client.leave id + afterCount = @_clientsInRoom(client, id) + logger.log {client: client.id, entity, id, beforeCount, afterCount}, "client left room" + # is the room now empty? if so, unsubscribe + if !entity? + logger.error {entity: id}, "unknown entity when leaving with id" + return + if beforeCount == 1 and afterCount == 0 + logger.log {entity, id}, "room is now empty" + RoomEvents.emit "#{entity}-empty", id + IdMap.delete(id) + # internal functions below, these access socket.io rooms data directly and # will need updating for socket.io v2 @@ -52,25 +77,3 @@ module.exports = RoomManager = [prefix, room] = fullRoomPath.split('/', 2) room return roomList - - _join: (client, entity, id) -> - beforeCount = @_clientsInRoom(client, id) - client.join id - afterCount = @_clientsInRoom(client, id) - logger.log {client: client.id, entity, id, beforeCount, afterCount}, "client joined room" - # is this a new room? if so, subscribe - if beforeCount == 0 and afterCount == 1 - logger.log {entity, id}, "room is now active" - RoomEvents.emit "#{entity}-active", id - IdMap.set(id, entity) - - _leave: (client, entity, id) -> - beforeCount = @_clientsInRoom(client, id) - client.leave id - afterCount = @_clientsInRoom(client, id) - logger.log {client: client.id, entity, id, beforeCount, afterCount}, "client left room" - # is the room now empty? if so, unsubscribe - if beforeCount == 1 and afterCount == 0 - logger.log {entity, id}, "room is now empty" - RoomEvents.emit "#{entity}-empty", id - IdMap.delete(id) \ No newline at end of file From 1afebd12a14cab488b920a8421e9fbf902bea796 Mon Sep 17 00:00:00 2001 From: Brian Gough Date: Mon, 22 Jul 2019 11:23:43 +0100 Subject: [PATCH 255/491] unit tests --- .../test/unit/coffee/ChannelManager.coffee | 93 +++++++++- .../test/unit/coffee/RoomManagerTests.coffee | 175 +++++++++++++++++- 2 files changed, 255 insertions(+), 13 deletions(-) diff --git a/services/real-time/test/unit/coffee/ChannelManager.coffee b/services/real-time/test/unit/coffee/ChannelManager.coffee index 314e9914e1..4ed852ddcf 100644 --- a/services/real-time/test/unit/coffee/ChannelManager.coffee +++ b/services/real-time/test/unit/coffee/ChannelManager.coffee @@ -6,28 +6,103 @@ SandboxedModule = require('sandboxed-module') describe 'ChannelManager', -> beforeEach -> - @project_id = "project-id-123" - @user_id = "user-id-123" - @user = {_id: @user_id} - @callback = sinon.stub() + @rclient = {} + @other_rclient = {} @ChannelManager = SandboxedModule.require modulePath, requires: "settings-sharelatex": @settings = {} + "metrics-sharelatex": @metrics = {inc: sinon.stub()} "logger-sharelatex": @logger = { log: sinon.stub(), error: sinon.stub() } describe "subscribe", -> - - describe "when the project room is empty", -> - describe "when there are other clients in the project room", -> + describe "when there is no existing subscription for this redis client", -> + beforeEach -> + @rclient.subscribe = sinon.stub() + @ChannelManager.subscribe @rclient, "applied-ops", "1234567890abcdef" + + it "should subscribe to the redis channel", -> + @rclient.subscribe.calledWithExactly("applied-ops:1234567890abcdef").should.equal true + + describe "when there is an existing subscription for this redis client", -> + beforeEach -> + @rclient.subscribe = sinon.stub() + @ChannelManager.subscribe @rclient, "applied-ops", "1234567890abcdef" + @rclient.subscribe = sinon.stub() # discard the original stub + @ChannelManager.subscribe @rclient, "applied-ops", "1234567890abcdef" + + it "should not subscribe to the redis channel", -> + @rclient.subscribe.called.should.equal false + + describe "when there is an existing subscription for another redis client but not this one", -> + beforeEach -> + @other_rclient.subscribe = sinon.stub() + @ChannelManager.subscribe @other_rclient, "applied-ops", "1234567890abcdef" + @rclient.subscribe = sinon.stub() # discard the original stub + @ChannelManager.subscribe @rclient, "applied-ops", "1234567890abcdef" + + it "should subscribe to the redis channel on this redis client", -> + @rclient.subscribe.calledWithExactly("applied-ops:1234567890abcdef").should.equal true describe "unsubscribe", -> - describe "when the doc room is empty", -> + describe "when there is no existing subscription for this redis client", -> + beforeEach -> + @rclient.unsubscribe = sinon.stub() + @ChannelManager.unsubscribe @rclient, "applied-ops", "1234567890abcdef" - describe "when there are other clients in the doc room", -> + it "should not unsubscribe from the redis channel", -> + @rclient.unsubscribe.called.should.equal false + + + describe "when there is an existing subscription for this another redis client but not this one", -> + beforeEach -> + @other_rclient.subscribe = sinon.stub() + @rclient.unsubscribe = sinon.stub() + @ChannelManager.subscribe @other_rclient, "applied-ops", "1234567890abcdef" + @ChannelManager.unsubscribe @rclient, "applied-ops", "1234567890abcdef" + + it "should not unsubscribe from the redis channel on this client", -> + @rclient.unsubscribe.called.should.equal false + + describe "when there is an existing subscription for this redis client", -> + beforeEach -> + @rclient.subscribe = sinon.stub() + @rclient.unsubscribe = sinon.stub() + @ChannelManager.subscribe @rclient, "applied-ops", "1234567890abcdef" + @ChannelManager.unsubscribe @rclient, "applied-ops", "1234567890abcdef" + + it "should unsubscribe from the redis channel", -> + @rclient.unsubscribe.calledWithExactly("applied-ops:1234567890abcdef").should.equal true describe "publish", -> describe "when the channel is 'all'", -> + beforeEach -> + @rclient.publish = sinon.stub() + @ChannelManager.publish @rclient, "applied-ops", "all", "random-message" + + it "should publish on the base channel", -> + @rclient.publish.calledWithExactly("applied-ops", "random-message").should.equal true describe "when the channel has an specific id", -> + + describe "when the individual channel setting is false", -> + beforeEach -> + @rclient.publish = sinon.stub() + @settings.publishOnIndividualChannels = false + @ChannelManager.publish @rclient, "applied-ops", "1234567890abcdef", "random-message" + + it "should publish on the per-id channel", -> + @rclient.publish.calledWithExactly("applied-ops", "random-message").should.equal true + @rclient.publish.calledOnce.should.equal true + + describe "when the individual channel setting is true", -> + beforeEach -> + @rclient.publish = sinon.stub() + @settings.publishOnIndividualChannels = true + @ChannelManager.publish @rclient, "applied-ops", "1234567890abcdef", "random-message" + + it "should publish on the per-id channel", -> + @rclient.publish.calledWithExactly("applied-ops:1234567890abcdef", "random-message").should.equal true + @rclient.publish.calledOnce.should.equal true + diff --git a/services/real-time/test/unit/coffee/RoomManagerTests.coffee b/services/real-time/test/unit/coffee/RoomManagerTests.coffee index 75d241cf6b..2f78b33c52 100644 --- a/services/real-time/test/unit/coffee/RoomManagerTests.coffee +++ b/services/real-time/test/unit/coffee/RoomManagerTests.coffee @@ -7,31 +7,198 @@ SandboxedModule = require('sandboxed-module') describe 'RoomManager', -> beforeEach -> @project_id = "project-id-123" - @user_id = "user-id-123" - @user = {_id: @user_id} - @callback = sinon.stub() + @doc_id = "doc-id-456" + @other_doc_id = "doc-id-789" + @client = {namespace: {name: ''}, id: "first-client"} @RoomManager = SandboxedModule.require modulePath, requires: "settings-sharelatex": @settings = {} "logger-sharelatex": @logger = { log: sinon.stub(), error: sinon.stub() } + @RoomManager._clientsInRoom = sinon.stub() + @RoomEvents = @RoomManager.eventSource() + sinon.spy(@RoomEvents, 'emit') describe "joinProject", -> describe "when the project room is empty", -> + beforeEach -> + @RoomManager._clientsInRoom + .withArgs(@client, @project_id) + .onFirstCall().returns(0) + .onSecondCall().returns(1) + @client.join = sinon.stub() + @RoomManager.joinProject @client, @project_id + + it "should join the room using the id", -> + @client.join.calledWithExactly(@project_id).should.equal true + + it "should emit a 'project-active' event with the id", -> + @RoomEvents.emit.calledWithExactly('project-active', @project_id).should.equal true + describe "when there are other clients in the project room", -> + beforeEach -> + @RoomManager._clientsInRoom + .withArgs(@client, @project_id) + .onFirstCall().returns(123) + .onSecondCall().returns(124) + @client.join = sinon.stub() + @RoomManager.joinProject @client, @project_id + + it "should join the room using the id", -> + @client.join.called.should.equal true + + it "should not emit any events", -> + @RoomEvents.emit.called.should.equal false + + describe "joinDoc", -> describe "when the doc room is empty", -> + beforeEach -> + @RoomManager._clientsInRoom + .withArgs(@client, @doc_id) + .onFirstCall().returns(0) + .onSecondCall().returns(1) + @client.join = sinon.stub() + @RoomManager.joinDoc @client, @doc_id + + it "should join the room using the id", -> + @client.join.calledWithExactly(@doc_id).should.equal true + + it "should emit a 'doc-active' event with the id", -> + @RoomEvents.emit.calledWithExactly('doc-active', @doc_id).should.equal true + describe "when there are other clients in the doc room", -> + beforeEach -> + @RoomManager._clientsInRoom + .withArgs(@client, @doc_id) + .onFirstCall().returns(123) + .onSecondCall().returns(124) + @client.join = sinon.stub() + @RoomManager.joinDoc @client, @doc_id + + it "should join the room using the id", -> + @client.join.called.should.equal true + + it "should not emit any events", -> + @RoomEvents.emit.called.should.equal false + + describe "leaveDoc", -> describe "when doc room will be empty after this client has left", -> + beforeEach -> + @RoomManager._clientsInRoom + .withArgs(@client, @doc_id) + .onFirstCall().returns(1) + .onSecondCall().returns(0) + @client.leave = sinon.stub() + @RoomManager.leaveDoc @client, @doc_id + + it "should leave the room using the id", -> + @client.leave.calledWithExactly(@doc_id).should.equal true + + it "should emit a 'doc-empty' event with the id", -> + @RoomEvents.emit.calledWithExactly('doc-empty', @doc_id).should.equal true + + describe "when there are other clients in the doc room", -> + beforeEach -> + @RoomManager._clientsInRoom + .withArgs(@client, @doc_id) + .onFirstCall().returns(123) + .onSecondCall().returns(122) + @client.leave = sinon.stub() + @RoomManager.leaveDoc @client, @doc_id + + it "should leave the room using the id", -> + @client.leave.calledWithExactly(@doc_id).should.equal true + + it "should not emit any events", -> + @RoomEvents.emit.called.should.equal false + + describe "leaveProjectAndDocs", -> - describe "when the client is connected to multiple docs", -> \ No newline at end of file + describe "when the client is connected to the project and multiple docs", -> + + beforeEach -> + @RoomManager._roomsClientIsIn = sinon.stub().returns [@project_id, @doc_id, @other_doc_id] + @client.join = sinon.stub() + @client.leave = sinon.stub() + + describe "when this is the only client connected", -> + + beforeEach -> + # first and secondc calls are for the join, + # calls 2 and 3 are for the leave + @RoomManager._clientsInRoom + .withArgs(@client, @doc_id) + .onCall(0).returns(0) + .onSecondCall().returns(1) + .onCall(2).returns(1) + .onCall(3).returns(0) + @RoomManager._clientsInRoom + .withArgs(@client, @other_doc_id) + .onCall(0).returns(0) + .onCall(1).returns(1) + .onCall(2).returns(1) + .onCall(3).returns(0) + @RoomManager._clientsInRoom + .withArgs(@client, @project_id) + .onCall(0).returns(0) + .onCall(1).returns(1) + .onCall(2).returns(1) + .onCall(3).returns(0) + # put the client in the rooms + @RoomManager.joinProject(@client, @project_id) + @RoomManager.joinDoc(@client, @doc_id) + @RoomManager.joinDoc(@client, @other_doc_id) + # now leave the project + @RoomManager.leaveProjectAndDocs @client + + it "should leave all the docs", -> + @client.leave.calledWithExactly(@doc_id).should.equal true + @client.leave.calledWithExactly(@other_doc_id).should.equal true + + it "should leave the project", -> + @client.leave.calledWithExactly(@project_id).should.equal true + + it "should emit a 'doc-empty' event with the id for each doc", -> + @RoomEvents.emit.calledWithExactly('doc-empty', @doc_id).should.equal true + @RoomEvents.emit.calledWithExactly('doc-empty', @other_doc_id).should.equal true + + it "should emit a 'project-empty' event with the id for the project", -> + @RoomEvents.emit.calledWithExactly('project-empty', @project_id).should.equal true + + describe "when other clients are still connected", -> + + beforeEach -> + @RoomManager._clientsInRoom + .withArgs(@client, @doc_id) + .onFirstCall().returns(123) + .onSecondCall().returns(122) + @RoomManager._clientsInRoom + .withArgs(@client, @other_doc_id) + .onFirstCall().returns(123) + .onSecondCall().returns(122) + @RoomManager._clientsInRoom + .withArgs(@client, @project_id) + .onFirstCall().returns(123) + .onSecondCall().returns(122) + @RoomManager.leaveProjectAndDocs @client + + it "should leave all the docs", -> + @client.leave.calledWithExactly(@doc_id).should.equal true + @client.leave.calledWithExactly(@other_doc_id).should.equal true + + it "should leave the project", -> + @client.leave.calledWithExactly(@project_id).should.equal true + + it "should not emit any events", -> + @RoomEvents.emit.called.should.equal false \ No newline at end of file From bb629c27a1936247ae47b217cadefeb6dbd65fe0 Mon Sep 17 00:00:00 2001 From: Brian Gough Date: Mon, 22 Jul 2019 11:28:49 +0100 Subject: [PATCH 256/491] rename unit test ChannelManager to ChannelManagerTests --- .../coffee/{ChannelManager.coffee => ChannelManagerTests.coffee} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename services/real-time/test/unit/coffee/{ChannelManager.coffee => ChannelManagerTests.coffee} (100%) diff --git a/services/real-time/test/unit/coffee/ChannelManager.coffee b/services/real-time/test/unit/coffee/ChannelManagerTests.coffee similarity index 100% rename from services/real-time/test/unit/coffee/ChannelManager.coffee rename to services/real-time/test/unit/coffee/ChannelManagerTests.coffee From 84e6ff616f46d7bcdb2712239ea4ba8bfccb106d Mon Sep 17 00:00:00 2001 From: Brian Gough Date: Mon, 22 Jul 2019 12:25:41 +0100 Subject: [PATCH 257/491] whitespace fix --- services/real-time/app/coffee/ChannelManager.coffee | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/services/real-time/app/coffee/ChannelManager.coffee b/services/real-time/app/coffee/ChannelManager.coffee index 1eca7a9143..0efef6ce96 100644 --- a/services/real-time/app/coffee/ChannelManager.coffee +++ b/services/real-time/app/coffee/ChannelManager.coffee @@ -43,4 +43,4 @@ module.exports = ChannelManager = channel = "#{baseChannel}:#{id}" # we publish on a different client to the subscribe, so we can't # check for the channel existing here - rclient.publish channel, data \ No newline at end of file + rclient.publish channel, data From 159b39c4915ec1821b39528e363014e30c8fba12 Mon Sep 17 00:00:00 2001 From: Brian Gough Date: Tue, 23 Jul 2019 17:02:09 +0100 Subject: [PATCH 258/491] ensure redis channel is subscribed when joining room --- .../app/coffee/ChannelManager.coffee | 13 ++- .../coffee/DocumentUpdaterController.coffee | 16 ++-- .../real-time/app/coffee/RoomManager.coffee | 28 +++--- .../app/coffee/WebsocketController.coffee | 14 +-- .../app/coffee/WebsocketLoadBalancer.coffee | 15 ++-- .../test/unit/coffee/RoomManagerTests.coffee | 88 ++++++++++++------- .../coffee/WebsocketControllerTests.coffee | 4 +- 7 files changed, 109 insertions(+), 69 deletions(-) diff --git a/services/real-time/app/coffee/ChannelManager.coffee b/services/real-time/app/coffee/ChannelManager.coffee index 0efef6ce96..900fd764f5 100644 --- a/services/real-time/app/coffee/ChannelManager.coffee +++ b/services/real-time/app/coffee/ChannelManager.coffee @@ -2,7 +2,7 @@ logger = require 'logger-sharelatex' metrics = require "metrics-sharelatex" settings = require "settings-sharelatex" -ClientMap = new Map() # for each redis client, stores a Set of subscribed channels +ClientMap = new Map() # for each redis client, stores a Map of subscribed channels (channelname -> subscribe promise) # Manage redis pubsub subscriptions for individual projects and docs, ensuring # that we never subscribe to a channel multiple times. The socket.io side is @@ -12,18 +12,23 @@ module.exports = ChannelManager = getClientMapEntry: (rclient) -> # return the rclient channel set if it exists, otherwise create and # return an empty set for the client. - ClientMap.get(rclient) || ClientMap.set(rclient, new Set()).get(rclient) + ClientMap.get(rclient) || ClientMap.set(rclient, new Map()).get(rclient) subscribe: (rclient, baseChannel, id) -> existingChannelSet = @getClientMapEntry(rclient) channel = "#{baseChannel}:#{id}" if existingChannelSet.has(channel) logger.error {channel}, "already subscribed - shouldn't happen" + # return the subscribe promise, so we can wait for it to resolve + return existingChannelSet.get(channel) else - rclient.subscribe channel # completes in the background - existingChannelSet.add(channel) + # get the subscribe promise and return it, the actual subscribe + # completes in the background + subscribePromise = rclient.subscribe channel + existingChannelSet.set(channel, subscribePromise) logger.log {channel}, "subscribed to new channel" metrics.inc "subscribe.#{baseChannel}" + return subscribePromise unsubscribe: (rclient, baseChannel, id) -> existingChannelSet = @getClientMapEntry(rclient) diff --git a/services/real-time/app/coffee/DocumentUpdaterController.coffee b/services/real-time/app/coffee/DocumentUpdaterController.coffee index 0cc5751d7c..e2d27fc343 100644 --- a/services/real-time/app/coffee/DocumentUpdaterController.coffee +++ b/services/real-time/app/coffee/DocumentUpdaterController.coffee @@ -7,6 +7,7 @@ HealthCheckManager = require "./HealthCheckManager" RoomManager = require "./RoomManager" ChannelManager = require "./ChannelManager" metrics = require "metrics-sharelatex" +util = require "util" MESSAGE_SIZE_LOG_LIMIT = 1024 * 1024 # 1Mb @@ -29,15 +30,20 @@ module.exports = DocumentUpdaterController = do (i) -> rclient.on "message", () -> metrics.inc "rclient-#{i}", 0.001 # per client event rate metric - for rclient in @rclientList - @handleRoomUpdates(rclient) + @handleRoomUpdates(@rclientList) - handleRoomUpdates: (rclientSub) -> + handleRoomUpdates: (rclientSubList) -> roomEvents = RoomManager.eventSource() roomEvents.on 'doc-active', (doc_id) -> - ChannelManager.subscribe rclientSub, "applied-ops", doc_id + subscribePromises = for rclient in rclientSubList + ChannelManager.subscribe rclient, "applied-ops", doc_id + subscribeResult = Promise.all(subscribePromises) + emitResult = (err) => this.emit("doc-subscribed-#{doc_id}", err) + subscribeResult.then () -> emitResult() + subscribeResult.catch (err) -> emitResult(err) roomEvents.on 'doc-empty', (doc_id) -> - ChannelManager.unsubscribe rclientSub, "applied-ops", doc_id + for rclient in rclientSubList + ChannelManager.unsubscribe rclient, "applied-ops", doc_id _processMessageFromDocumentUpdater: (io, channel, message) -> SafeJsonParse.parse message, (error, message) -> diff --git a/services/real-time/app/coffee/RoomManager.coffee b/services/real-time/app/coffee/RoomManager.coffee index 225dd37f6d..08e65b5e52 100644 --- a/services/real-time/app/coffee/RoomManager.coffee +++ b/services/real-time/app/coffee/RoomManager.coffee @@ -16,11 +16,11 @@ RoomEvents = new EventEmitter() module.exports = RoomManager = - joinProject: (client, project_id) -> - @joinEntity client, "project", project_id + joinProject: (client, project_id, callback = () ->) -> + @joinEntity client, "project", project_id, callback - joinDoc: (client, doc_id) -> - @joinEntity client, "doc", doc_id + joinDoc: (client, doc_id, callback = () ->) -> + @joinEntity client, "doc", doc_id, callback leaveDoc: (client, doc_id) -> @leaveEntity client, "doc", doc_id @@ -38,27 +38,31 @@ module.exports = RoomManager = eventSource: () -> return RoomEvents - joinEntity: (client, entity, id) -> + joinEntity: (client, entity, id, callback) -> beforeCount = @_clientsInRoom(client, id) - client.join id - afterCount = @_clientsInRoom(client, id) - logger.log {client: client.id, entity, id, beforeCount, afterCount}, "client joined room" # is this a new room? if so, subscribe - if beforeCount == 0 and afterCount == 1 + if beforeCount == 0 logger.log {entity, id}, "room is now active" + RoomEvents.once "#{entity}-subscribed-#{id}", (err) -> + logger.log {client: client.id, entity, id, beforeCount}, "client joined room after subscribing channel" + client.join id + callback(err) RoomEvents.emit "#{entity}-active", id IdMap.set(id, entity) + else + logger.log {client: client.id, entity, id, beforeCount}, "client joined existing room" + client.join id + callback() leaveEntity: (client, entity, id) -> - beforeCount = @_clientsInRoom(client, id) client.leave id afterCount = @_clientsInRoom(client, id) - logger.log {client: client.id, entity, id, beforeCount, afterCount}, "client left room" + logger.log {client: client.id, entity, id, afterCount}, "client left room" # is the room now empty? if so, unsubscribe if !entity? logger.error {entity: id}, "unknown entity when leaving with id" return - if beforeCount == 1 and afterCount == 0 + if afterCount == 0 logger.log {entity, id}, "room is now empty" RoomEvents.emit "#{entity}-empty", id IdMap.delete(id) diff --git a/services/real-time/app/coffee/WebsocketController.coffee b/services/real-time/app/coffee/WebsocketController.coffee index 6d8965883f..22ea7e7a0c 100644 --- a/services/real-time/app/coffee/WebsocketController.coffee +++ b/services/real-time/app/coffee/WebsocketController.coffee @@ -25,8 +25,7 @@ module.exports = WebsocketController = err = new Error("not authorized") logger.warn {err, project_id, user_id, client_id: client.id}, "user is not authorized to join project" return callback(err) - - RoomManager.joinProject(client, project_id) + client.set("privilege_level", privilegeLevel) client.set("user_id", user_id) @@ -39,8 +38,9 @@ module.exports = WebsocketController = client.set("signup_date", user?.signUpDate) client.set("login_count", user?.loginCount) - callback null, project, privilegeLevel, WebsocketController.PROTOCOL_VERSION - logger.log {user_id, project_id, client_id: client.id}, "user joined project" + RoomManager.joinProject client, project_id, (err) -> + logger.log {user_id, project_id, client_id: client.id}, "user joined project" + callback null, project, privilegeLevel, WebsocketController.PROTOCOL_VERSION # No need to block for setting the user as connected in the cursor tracking ConnectedUsersManager.updateUserPosition project_id, client.id, user, null, () -> @@ -118,9 +118,9 @@ module.exports = WebsocketController = return callback(err) AuthorizationManager.addAccessToDoc client, doc_id - RoomManager.joinDoc(client, doc_id) - callback null, escapedLines, version, ops, ranges - logger.log {user_id, project_id, doc_id, fromVersion, client_id: client.id}, "client joined doc" + RoomManager.joinDoc client, doc_id, (err) -> + logger.log {user_id, project_id, doc_id, fromVersion, client_id: client.id}, "client joined doc" + callback null, escapedLines, version, ops, ranges leaveDoc: (client, doc_id, callback = (error) ->) -> metrics.inc "editor.leave-doc" diff --git a/services/real-time/app/coffee/WebsocketLoadBalancer.coffee b/services/real-time/app/coffee/WebsocketLoadBalancer.coffee index 1bb74c6a3e..e8ff88aa7b 100644 --- a/services/real-time/app/coffee/WebsocketLoadBalancer.coffee +++ b/services/real-time/app/coffee/WebsocketLoadBalancer.coffee @@ -35,15 +35,20 @@ module.exports = WebsocketLoadBalancer = rclientSub.on "message", (channel, message) -> EventLogger.debugEvent(channel, message) if Settings.debugEvents > 0 WebsocketLoadBalancer._processEditorEvent io, channel, message - for rclientSub in @rclientSubList - @handleRoomUpdates(rclientSub) + @handleRoomUpdates(@rclientSubList) - handleRoomUpdates: (rclientSub) -> + handleRoomUpdates: (rclientSubList) -> roomEvents = RoomManager.eventSource() roomEvents.on 'project-active', (project_id) -> - ChannelManager.subscribe rclientSub, "editor-events", project_id + subscribePromises = for rclient in rclientSubList + ChannelManager.subscribe rclient, "editor-events", project_id + subscribeResult = Promise.all(subscribePromises) + emitResult = (err) => this.emit("project-subscribed-#{project_id}", err) + subscribeResult.then () -> emitResult() + subscribeResult.catch (err) -> emitResult(err) roomEvents.on 'project-empty', (project_id) -> - ChannelManager.unsubscribe rclientSub, "editor-events", project_id + for rclient in rclientSubList + ChannelManager.unsubscribe rclient, "editor-events", project_id _processEditorEvent: (io, channel, message) -> SafeJsonParse.parse message, (error, message) -> diff --git a/services/real-time/test/unit/coffee/RoomManagerTests.coffee b/services/real-time/test/unit/coffee/RoomManagerTests.coffee index 2f78b33c52..f9fde4dd83 100644 --- a/services/real-time/test/unit/coffee/RoomManagerTests.coffee +++ b/services/real-time/test/unit/coffee/RoomManagerTests.coffee @@ -15,26 +15,36 @@ describe 'RoomManager', -> "logger-sharelatex": @logger = { log: sinon.stub(), error: sinon.stub() } @RoomManager._clientsInRoom = sinon.stub() @RoomEvents = @RoomManager.eventSource() - sinon.spy(@RoomEvents, 'emit') + sinon.spy(@RoomEvents, 'emit') + sinon.spy(@RoomEvents, 'once') describe "joinProject", -> describe "when the project room is empty", -> - beforeEach -> + beforeEach (done) -> @RoomManager._clientsInRoom .withArgs(@client, @project_id) .onFirstCall().returns(0) - .onSecondCall().returns(1) @client.join = sinon.stub() - @RoomManager.joinProject @client, @project_id - - it "should join the room using the id", -> - @client.join.calledWithExactly(@project_id).should.equal true + @callback = sinon.stub() + @RoomEvents.on 'project-active', (id) => + setTimeout () => + @RoomEvents.emit "project-subscribed-#{id}" + , 100 + @RoomManager.joinProject @client, @project_id, (err) => + @callback(err) + done() it "should emit a 'project-active' event with the id", -> @RoomEvents.emit.calledWithExactly('project-active', @project_id).should.equal true + it "should listen for the 'project-subscribed-id' event", -> + @RoomEvents.once.calledWith("project-subscribed-#{@project_id}").should.equal true + + it "should join the room using the id", -> + @client.join.calledWithExactly(@project_id).should.equal true + describe "when there are other clients in the project room", -> beforeEach -> @@ -56,20 +66,29 @@ describe 'RoomManager', -> describe "when the doc room is empty", -> - beforeEach -> + beforeEach (done) -> @RoomManager._clientsInRoom .withArgs(@client, @doc_id) .onFirstCall().returns(0) - .onSecondCall().returns(1) @client.join = sinon.stub() - @RoomManager.joinDoc @client, @doc_id - - it "should join the room using the id", -> - @client.join.calledWithExactly(@doc_id).should.equal true + @callback = sinon.stub() + @RoomEvents.on 'doc-active', (id) => + setTimeout () => + @RoomEvents.emit "doc-subscribed-#{id}" + , 100 + @RoomManager.joinDoc @client, @doc_id, (err) => + @callback(err) + done() it "should emit a 'doc-active' event with the id", -> @RoomEvents.emit.calledWithExactly('doc-active', @doc_id).should.equal true + it "should listen for the 'doc-subscribed-id' event", -> + @RoomEvents.once.calledWith("doc-subscribed-#{@doc_id}").should.equal true + + it "should join the room using the id", -> + @client.join.calledWithExactly(@doc_id).should.equal true + describe "when there are other clients in the doc room", -> beforeEach -> @@ -94,8 +113,7 @@ describe 'RoomManager', -> beforeEach -> @RoomManager._clientsInRoom .withArgs(@client, @doc_id) - .onFirstCall().returns(1) - .onSecondCall().returns(0) + .onCall(0).returns(0) @client.leave = sinon.stub() @RoomManager.leaveDoc @client, @doc_id @@ -111,8 +129,7 @@ describe 'RoomManager', -> beforeEach -> @RoomManager._clientsInRoom .withArgs(@client, @doc_id) - .onFirstCall().returns(123) - .onSecondCall().returns(122) + .onCall(0).returns(123) @client.leave = sinon.stub() @RoomManager.leaveDoc @client, @doc_id @@ -134,33 +151,36 @@ describe 'RoomManager', -> describe "when this is the only client connected", -> - beforeEach -> - # first and secondc calls are for the join, - # calls 2 and 3 are for the leave + beforeEach (done) -> + # first call is for the join, + # second for the leave @RoomManager._clientsInRoom .withArgs(@client, @doc_id) .onCall(0).returns(0) - .onSecondCall().returns(1) - .onCall(2).returns(1) - .onCall(3).returns(0) + .onCall(1).returns(0) @RoomManager._clientsInRoom .withArgs(@client, @other_doc_id) .onCall(0).returns(0) - .onCall(1).returns(1) - .onCall(2).returns(1) - .onCall(3).returns(0) + .onCall(1).returns(0) @RoomManager._clientsInRoom .withArgs(@client, @project_id) .onCall(0).returns(0) - .onCall(1).returns(1) - .onCall(2).returns(1) - .onCall(3).returns(0) + .onCall(1).returns(0) + @RoomEvents.on 'project-active', (id) => + setTimeout () => + @RoomEvents.emit "project-subscribed-#{id}" + , 100 + @RoomEvents.on 'doc-active', (id) => + setTimeout () => + @RoomEvents.emit "doc-subscribed-#{id}" + , 100 # put the client in the rooms - @RoomManager.joinProject(@client, @project_id) - @RoomManager.joinDoc(@client, @doc_id) - @RoomManager.joinDoc(@client, @other_doc_id) - # now leave the project - @RoomManager.leaveProjectAndDocs @client + @RoomManager.joinProject @client, @project_id, () => + @RoomManager.joinDoc @client, @doc_id, () => + @RoomManager.joinDoc @client, @other_doc_id, () => + # now leave the project + @RoomManager.leaveProjectAndDocs @client + done() it "should leave all the docs", -> @client.leave.calledWithExactly(@doc_id).should.equal true diff --git a/services/real-time/test/unit/coffee/WebsocketControllerTests.coffee b/services/real-time/test/unit/coffee/WebsocketControllerTests.coffee index ab442006c2..d0dad108e7 100644 --- a/services/real-time/test/unit/coffee/WebsocketControllerTests.coffee +++ b/services/real-time/test/unit/coffee/WebsocketControllerTests.coffee @@ -54,7 +54,7 @@ describe 'WebsocketController', -> @privilegeLevel = "owner" @ConnectedUsersManager.updateUserPosition = sinon.stub().callsArg(4) @WebApiManager.joinProject = sinon.stub().callsArgWith(2, null, @project, @privilegeLevel) - @RoomManager.joinProject = sinon.stub() + @RoomManager.joinProject = sinon.stub().callsArg(2) @WebsocketController.joinProject @client, @user, @project_id, @callback it "should load the project from web", -> @@ -237,7 +237,7 @@ describe 'WebsocketController', -> @AuthorizationManager.addAccessToDoc = sinon.stub() @AuthorizationManager.assertClientCanViewProject = sinon.stub().callsArgWith(1, null) @DocumentUpdaterManager.getDocument = sinon.stub().callsArgWith(3, null, @doc_lines, @version, @ranges, @ops) - @RoomManager.joinDoc = sinon.stub() + @RoomManager.joinDoc = sinon.stub().callsArg(2) describe "works", -> beforeEach -> From 61b3a000b406de1c3e77c786c0d5b70480641246 Mon Sep 17 00:00:00 2001 From: Brian Gough Date: Wed, 24 Jul 2019 09:52:20 +0100 Subject: [PATCH 259/491] fix whitespace --- services/real-time/app/coffee/ChannelManager.coffee | 2 +- services/real-time/test/unit/coffee/RoomManagerTests.coffee | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/services/real-time/app/coffee/ChannelManager.coffee b/services/real-time/app/coffee/ChannelManager.coffee index 900fd764f5..8cec24156d 100644 --- a/services/real-time/app/coffee/ChannelManager.coffee +++ b/services/real-time/app/coffee/ChannelManager.coffee @@ -24,7 +24,7 @@ module.exports = ChannelManager = else # get the subscribe promise and return it, the actual subscribe # completes in the background - subscribePromise = rclient.subscribe channel + subscribePromise = rclient.subscribe channel existingChannelSet.set(channel, subscribePromise) logger.log {channel}, "subscribed to new channel" metrics.inc "subscribe.#{baseChannel}" diff --git a/services/real-time/test/unit/coffee/RoomManagerTests.coffee b/services/real-time/test/unit/coffee/RoomManagerTests.coffee index f9fde4dd83..294365dd61 100644 --- a/services/real-time/test/unit/coffee/RoomManagerTests.coffee +++ b/services/real-time/test/unit/coffee/RoomManagerTests.coffee @@ -16,7 +16,7 @@ describe 'RoomManager', -> @RoomManager._clientsInRoom = sinon.stub() @RoomEvents = @RoomManager.eventSource() sinon.spy(@RoomEvents, 'emit') - sinon.spy(@RoomEvents, 'once') + sinon.spy(@RoomEvents, 'once') describe "joinProject", -> @@ -152,7 +152,7 @@ describe 'RoomManager', -> describe "when this is the only client connected", -> beforeEach (done) -> - # first call is for the join, + # first call is for the join, # second for the leave @RoomManager._clientsInRoom .withArgs(@client, @doc_id) From cb53bfafd6038c28123bc3e43d1bfcb600b7b347 Mon Sep 17 00:00:00 2001 From: Brian Gough Date: Wed, 24 Jul 2019 09:52:31 +0100 Subject: [PATCH 260/491] remove unnecessary require --- services/real-time/app/coffee/DocumentUpdaterController.coffee | 1 - 1 file changed, 1 deletion(-) diff --git a/services/real-time/app/coffee/DocumentUpdaterController.coffee b/services/real-time/app/coffee/DocumentUpdaterController.coffee index e2d27fc343..6d6ec4b79d 100644 --- a/services/real-time/app/coffee/DocumentUpdaterController.coffee +++ b/services/real-time/app/coffee/DocumentUpdaterController.coffee @@ -7,7 +7,6 @@ HealthCheckManager = require "./HealthCheckManager" RoomManager = require "./RoomManager" ChannelManager = require "./ChannelManager" metrics = require "metrics-sharelatex" -util = require "util" MESSAGE_SIZE_LOG_LIMIT = 1024 * 1024 # 1Mb From e14a94906aca668af5593f0fe73b0682d6ab5bfd Mon Sep 17 00:00:00 2001 From: Brian Gough Date: Wed, 24 Jul 2019 14:17:19 +0100 Subject: [PATCH 261/491] update naming from Set -> Map --- .../app/coffee/ChannelManager.coffee | 24 +++++++++---------- .../unit/coffee/ChannelManagerTests.coffee | 2 +- 2 files changed, 13 insertions(+), 13 deletions(-) diff --git a/services/real-time/app/coffee/ChannelManager.coffee b/services/real-time/app/coffee/ChannelManager.coffee index 8cec24156d..1749ab2d58 100644 --- a/services/real-time/app/coffee/ChannelManager.coffee +++ b/services/real-time/app/coffee/ChannelManager.coffee @@ -2,7 +2,7 @@ logger = require 'logger-sharelatex' metrics = require "metrics-sharelatex" settings = require "settings-sharelatex" -ClientMap = new Map() # for each redis client, stores a Map of subscribed channels (channelname -> subscribe promise) +ClientMap = new Map() # for each redis client, store a Map of subscribed channels (channelname -> subscribe promise) # Manage redis pubsub subscriptions for individual projects and docs, ensuring # that we never subscribe to a channel multiple times. The socket.io side is @@ -10,34 +10,34 @@ ClientMap = new Map() # for each redis client, stores a Map of subscribed channe module.exports = ChannelManager = getClientMapEntry: (rclient) -> - # return the rclient channel set if it exists, otherwise create and - # return an empty set for the client. + # return the per-client channel map if it exists, otherwise create and + # return an empty map for the client. ClientMap.get(rclient) || ClientMap.set(rclient, new Map()).get(rclient) subscribe: (rclient, baseChannel, id) -> - existingChannelSet = @getClientMapEntry(rclient) + clientChannelMap = @getClientMapEntry(rclient) channel = "#{baseChannel}:#{id}" - if existingChannelSet.has(channel) - logger.error {channel}, "already subscribed - shouldn't happen" - # return the subscribe promise, so we can wait for it to resolve - return existingChannelSet.get(channel) + if clientChannelMap.has(channel) + logger.warn {channel}, "subscribe already actioned" + # return the existing subscribe promise, so we can wait for it to resolve + return clientChannelMap.get(channel) else # get the subscribe promise and return it, the actual subscribe # completes in the background subscribePromise = rclient.subscribe channel - existingChannelSet.set(channel, subscribePromise) + clientChannelMap.set(channel, subscribePromise) logger.log {channel}, "subscribed to new channel" metrics.inc "subscribe.#{baseChannel}" return subscribePromise unsubscribe: (rclient, baseChannel, id) -> - existingChannelSet = @getClientMapEntry(rclient) + clientChannelMap = @getClientMapEntry(rclient) channel = "#{baseChannel}:#{id}" - if !existingChannelSet.has(channel) + if !clientChannelMap.has(channel) logger.error {channel}, "not subscribed - shouldn't happen" else rclient.unsubscribe channel # completes in the background - existingChannelSet.delete(channel) + clientChannelMap.delete(channel) logger.log {channel}, "unsubscribed from channel" metrics.inc "unsubscribe.#{baseChannel}" diff --git a/services/real-time/test/unit/coffee/ChannelManagerTests.coffee b/services/real-time/test/unit/coffee/ChannelManagerTests.coffee index 4ed852ddcf..e550e963d4 100644 --- a/services/real-time/test/unit/coffee/ChannelManagerTests.coffee +++ b/services/real-time/test/unit/coffee/ChannelManagerTests.coffee @@ -11,7 +11,7 @@ describe 'ChannelManager', -> @ChannelManager = SandboxedModule.require modulePath, requires: "settings-sharelatex": @settings = {} "metrics-sharelatex": @metrics = {inc: sinon.stub()} - "logger-sharelatex": @logger = { log: sinon.stub(), error: sinon.stub() } + "logger-sharelatex": @logger = { log: sinon.stub(), warn: sinon.stub(), error: sinon.stub() } describe "subscribe", -> From 273af3f3aad574b17e9602ba5efc6077c0bf21f6 Mon Sep 17 00:00:00 2001 From: Brian Gough Date: Wed, 24 Jul 2019 14:30:48 +0100 Subject: [PATCH 262/491] refactor subscribe resolution --- .../real-time/app/coffee/DocumentUpdaterController.coffee | 5 +---- services/real-time/app/coffee/RoomManager.coffee | 5 +++++ services/real-time/app/coffee/WebsocketLoadBalancer.coffee | 5 +---- 3 files changed, 7 insertions(+), 8 deletions(-) diff --git a/services/real-time/app/coffee/DocumentUpdaterController.coffee b/services/real-time/app/coffee/DocumentUpdaterController.coffee index 6d6ec4b79d..2611d484ad 100644 --- a/services/real-time/app/coffee/DocumentUpdaterController.coffee +++ b/services/real-time/app/coffee/DocumentUpdaterController.coffee @@ -36,10 +36,7 @@ module.exports = DocumentUpdaterController = roomEvents.on 'doc-active', (doc_id) -> subscribePromises = for rclient in rclientSubList ChannelManager.subscribe rclient, "applied-ops", doc_id - subscribeResult = Promise.all(subscribePromises) - emitResult = (err) => this.emit("doc-subscribed-#{doc_id}", err) - subscribeResult.then () -> emitResult() - subscribeResult.catch (err) -> emitResult(err) + RoomManager.emitOnCompletion(subscribePromises, "doc-subscribed-#{doc_id}") roomEvents.on 'doc-empty', (doc_id) -> for rclient in rclientSubList ChannelManager.unsubscribe rclient, "applied-ops", doc_id diff --git a/services/real-time/app/coffee/RoomManager.coffee b/services/real-time/app/coffee/RoomManager.coffee index 08e65b5e52..152a290020 100644 --- a/services/real-time/app/coffee/RoomManager.coffee +++ b/services/real-time/app/coffee/RoomManager.coffee @@ -35,6 +35,11 @@ module.exports = RoomManager = entity = IdMap.get(id) @leaveEntity client, entity, id + emitOnCompletion: (promiseList, eventName) -> + result = Promise.all(promiseList) + result.then () -> RoomEvents.emit(eventName) + result.catch (err) -> RoomEvents.emit(eventName, err) + eventSource: () -> return RoomEvents diff --git a/services/real-time/app/coffee/WebsocketLoadBalancer.coffee b/services/real-time/app/coffee/WebsocketLoadBalancer.coffee index e8ff88aa7b..865249c63e 100644 --- a/services/real-time/app/coffee/WebsocketLoadBalancer.coffee +++ b/services/real-time/app/coffee/WebsocketLoadBalancer.coffee @@ -42,10 +42,7 @@ module.exports = WebsocketLoadBalancer = roomEvents.on 'project-active', (project_id) -> subscribePromises = for rclient in rclientSubList ChannelManager.subscribe rclient, "editor-events", project_id - subscribeResult = Promise.all(subscribePromises) - emitResult = (err) => this.emit("project-subscribed-#{project_id}", err) - subscribeResult.then () -> emitResult() - subscribeResult.catch (err) -> emitResult(err) + RoomManager.emitOnCompletion(subscribePromises, "project-subscribed-#{project_id}") roomEvents.on 'project-empty', (project_id) -> for rclient in rclientSubList ChannelManager.unsubscribe rclient, "editor-events", project_id From 1c74cbbc4ef21d5531d2486a2a57829a7e3546d6 Mon Sep 17 00:00:00 2001 From: Brian Gough Date: Wed, 24 Jul 2019 15:41:25 +0100 Subject: [PATCH 263/491] add comments --- services/real-time/app/coffee/ChannelManager.coffee | 6 ++++++ services/real-time/app/coffee/RoomManager.coffee | 3 ++- 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/services/real-time/app/coffee/ChannelManager.coffee b/services/real-time/app/coffee/ChannelManager.coffee index 1749ab2d58..3ea5c2e71e 100644 --- a/services/real-time/app/coffee/ChannelManager.coffee +++ b/services/real-time/app/coffee/ChannelManager.coffee @@ -17,6 +17,9 @@ module.exports = ChannelManager = subscribe: (rclient, baseChannel, id) -> clientChannelMap = @getClientMapEntry(rclient) channel = "#{baseChannel}:#{id}" + # we track pending subscribes because we want to be sure that the + # channel is active before letting the client join the doc or project, + # so that events are not lost. if clientChannelMap.has(channel) logger.warn {channel}, "subscribe already actioned" # return the existing subscribe promise, so we can wait for it to resolve @@ -33,6 +36,9 @@ module.exports = ChannelManager = unsubscribe: (rclient, baseChannel, id) -> clientChannelMap = @getClientMapEntry(rclient) channel = "#{baseChannel}:#{id}" + # we don't need to track pending unsubscribes, because we there is no + # harm if events continue to arrive on the channel while the unsubscribe + # command in pending. if !clientChannelMap.has(channel) logger.error {channel}, "not subscribed - shouldn't happen" else diff --git a/services/real-time/app/coffee/RoomManager.coffee b/services/real-time/app/coffee/RoomManager.coffee index 152a290020..9f42083198 100644 --- a/services/real-time/app/coffee/RoomManager.coffee +++ b/services/real-time/app/coffee/RoomManager.coffee @@ -2,7 +2,7 @@ logger = require 'logger-sharelatex' {EventEmitter} = require 'events' IdMap = new Map() # keep track of whether ids are from projects or docs -RoomEvents = new EventEmitter() +RoomEvents = new EventEmitter() # emits {project,doc}-active and {project,doc}-empty events # Manage socket.io rooms for individual projects and docs # @@ -49,6 +49,7 @@ module.exports = RoomManager = if beforeCount == 0 logger.log {entity, id}, "room is now active" RoomEvents.once "#{entity}-subscribed-#{id}", (err) -> + # only allow the client to join when all the relevant channels have subscribed logger.log {client: client.id, entity, id, beforeCount}, "client joined room after subscribing channel" client.join id callback(err) From 277ec71a5b7db19259dee339897c07fdb6f7beec Mon Sep 17 00:00:00 2001 From: Brian Gough Date: Wed, 24 Jul 2019 15:43:48 +0100 Subject: [PATCH 264/491] subscribe to doc updates before requesting doc content --- .../app/coffee/WebsocketController.coffee | 51 ++++++++++--------- 1 file changed, 27 insertions(+), 24 deletions(-) diff --git a/services/real-time/app/coffee/WebsocketController.coffee b/services/real-time/app/coffee/WebsocketController.coffee index 22ea7e7a0c..fce505c1bc 100644 --- a/services/real-time/app/coffee/WebsocketController.coffee +++ b/services/real-time/app/coffee/WebsocketController.coffee @@ -92,33 +92,36 @@ module.exports = WebsocketController = AuthorizationManager.assertClientCanViewProject client, (error) -> return callback(error) if error? - DocumentUpdaterManager.getDocument project_id, doc_id, fromVersion, (error, lines, version, ranges, ops) -> + # ensure the per-doc applied-ops channel is subscribed before sending the + # doc to the client, so that no events are missed. + RoomManager.joinDoc client, doc_id, (error) -> return callback(error) if error? + DocumentUpdaterManager.getDocument project_id, doc_id, fromVersion, (error, lines, version, ranges, ops) -> + return callback(error) if error? - # Encode any binary bits of data so it can go via WebSockets - # See http://ecmanaut.blogspot.co.uk/2006/07/encoding-decoding-utf8-in-javascript.html - encodeForWebsockets = (text) -> unescape(encodeURIComponent(text)) - escapedLines = [] - for line in lines - try - line = encodeForWebsockets(line) - catch err - logger.err {err, project_id, doc_id, fromVersion, line, client_id: client.id}, "error encoding line uri component" - return callback(err) - escapedLines.push line - if options.encodeRanges - try - for comment in ranges?.comments or [] - comment.op.c = encodeForWebsockets(comment.op.c) if comment.op.c? - for change in ranges?.changes or [] - change.op.i = encodeForWebsockets(change.op.i) if change.op.i? - change.op.d = encodeForWebsockets(change.op.d) if change.op.d? - catch err - logger.err {err, project_id, doc_id, fromVersion, ranges, client_id: client.id}, "error encoding range uri component" - return callback(err) + # Encode any binary bits of data so it can go via WebSockets + # See http://ecmanaut.blogspot.co.uk/2006/07/encoding-decoding-utf8-in-javascript.html + encodeForWebsockets = (text) -> unescape(encodeURIComponent(text)) + escapedLines = [] + for line in lines + try + line = encodeForWebsockets(line) + catch err + logger.err {err, project_id, doc_id, fromVersion, line, client_id: client.id}, "error encoding line uri component" + return callback(err) + escapedLines.push line + if options.encodeRanges + try + for comment in ranges?.comments or [] + comment.op.c = encodeForWebsockets(comment.op.c) if comment.op.c? + for change in ranges?.changes or [] + change.op.i = encodeForWebsockets(change.op.i) if change.op.i? + change.op.d = encodeForWebsockets(change.op.d) if change.op.d? + catch err + logger.err {err, project_id, doc_id, fromVersion, ranges, client_id: client.id}, "error encoding range uri component" + return callback(err) - AuthorizationManager.addAccessToDoc client, doc_id - RoomManager.joinDoc client, doc_id, (err) -> + AuthorizationManager.addAccessToDoc client, doc_id logger.log {user_id, project_id, doc_id, fromVersion, client_id: client.id}, "client joined doc" callback null, escapedLines, version, ops, ranges From 22d722f3e8afca3ebdf773d9d57a1ef83f1ac706 Mon Sep 17 00:00:00 2001 From: Brian Gough Date: Wed, 24 Jul 2019 16:25:45 +0100 Subject: [PATCH 265/491] add metric for RoomEvents listeners --- services/real-time/app/coffee/RoomManager.coffee | 4 ++++ services/real-time/test/unit/coffee/RoomManagerTests.coffee | 1 + 2 files changed, 5 insertions(+) diff --git a/services/real-time/app/coffee/RoomManager.coffee b/services/real-time/app/coffee/RoomManager.coffee index 9f42083198..adf472e26c 100644 --- a/services/real-time/app/coffee/RoomManager.coffee +++ b/services/real-time/app/coffee/RoomManager.coffee @@ -1,4 +1,5 @@ logger = require 'logger-sharelatex' +metrics = require "metrics-sharelatex" {EventEmitter} = require 'events' IdMap = new Map() # keep track of whether ids are from projects or docs @@ -55,6 +56,8 @@ module.exports = RoomManager = callback(err) RoomEvents.emit "#{entity}-active", id IdMap.set(id, entity) + # keep track of the number of listeners + metrics.gauge "room-listeners", RoomEvents.eventNames().length else logger.log {client: client.id, entity, id, beforeCount}, "client joined existing room" client.join id @@ -72,6 +75,7 @@ module.exports = RoomManager = logger.log {entity, id}, "room is now empty" RoomEvents.emit "#{entity}-empty", id IdMap.delete(id) + metrics.gauge "room-listeners", RoomEvents.eventNames().length # internal functions below, these access socket.io rooms data directly and # will need updating for socket.io v2 diff --git a/services/real-time/test/unit/coffee/RoomManagerTests.coffee b/services/real-time/test/unit/coffee/RoomManagerTests.coffee index 294365dd61..ee46a3ef04 100644 --- a/services/real-time/test/unit/coffee/RoomManagerTests.coffee +++ b/services/real-time/test/unit/coffee/RoomManagerTests.coffee @@ -13,6 +13,7 @@ describe 'RoomManager', -> @RoomManager = SandboxedModule.require modulePath, requires: "settings-sharelatex": @settings = {} "logger-sharelatex": @logger = { log: sinon.stub(), error: sinon.stub() } + "metrics-sharelatex": @metrics = { gauge: sinon.stub() } @RoomManager._clientsInRoom = sinon.stub() @RoomEvents = @RoomManager.eventSource() sinon.spy(@RoomEvents, 'emit') From cf0df28f4c05b01f0b2284a8d01b39f5d796621e Mon Sep 17 00:00:00 2001 From: mserranom Date: Thu, 25 Jul 2019 09:22:24 +0000 Subject: [PATCH 266/491] Patched EventEmitter for socket.io compatibility with Node >= 7 --- services/real-time/socket.io.patch.js | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/services/real-time/socket.io.patch.js b/services/real-time/socket.io.patch.js index 21a9608133..753f44b8ca 100644 --- a/services/real-time/socket.io.patch.js +++ b/services/real-time/socket.io.patch.js @@ -1,3 +1,9 @@ +// EventEmitter has been removed from process in node >= 7 +// https://github.com/nodejs/node/commit/62b544290a075fe38e233887a06c408ba25a1c71 +if(process.versions.node.split('.')[0] >= 7) { + process.EventEmitter = require('events') +} + var io = require("socket.io"); if (io.version === "0.9.16") { From 04a171171f761b08a50d3c7a15010753a86230d8 Mon Sep 17 00:00:00 2001 From: Brian Gough Date: Fri, 26 Jul 2019 08:07:49 +0100 Subject: [PATCH 267/491] fix async behaviour of join/leave --- .../real-time/app/coffee/RoomManager.coffee | 6 +++-- .../acceptance/coffee/LeaveDocTests.coffee | 26 +++++++++++++++++-- 2 files changed, 28 insertions(+), 4 deletions(-) diff --git a/services/real-time/app/coffee/RoomManager.coffee b/services/real-time/app/coffee/RoomManager.coffee index adf472e26c..0d8a8aaf06 100644 --- a/services/real-time/app/coffee/RoomManager.coffee +++ b/services/real-time/app/coffee/RoomManager.coffee @@ -46,13 +46,15 @@ module.exports = RoomManager = joinEntity: (client, entity, id, callback) -> beforeCount = @_clientsInRoom(client, id) + # client joins room immediately but joinDoc request does not complete + # until room is subscribed + client.join id # is this a new room? if so, subscribe if beforeCount == 0 logger.log {entity, id}, "room is now active" RoomEvents.once "#{entity}-subscribed-#{id}", (err) -> # only allow the client to join when all the relevant channels have subscribed - logger.log {client: client.id, entity, id, beforeCount}, "client joined room after subscribing channel" - client.join id + logger.log {client: client.id, entity, id, beforeCount}, "client joined new room and subscribed to channel" callback(err) RoomEvents.emit "#{entity}-active", id IdMap.set(id, entity) diff --git a/services/real-time/test/acceptance/coffee/LeaveDocTests.coffee b/services/real-time/test/acceptance/coffee/LeaveDocTests.coffee index 753bc79c62..e4875fcbe5 100644 --- a/services/real-time/test/acceptance/coffee/LeaveDocTests.coffee +++ b/services/real-time/test/acceptance/coffee/LeaveDocTests.coffee @@ -1,10 +1,12 @@ chai = require("chai") expect = chai.expect chai.should() +sinon = require("sinon") RealTimeClient = require "./helpers/RealTimeClient" MockDocUpdaterServer = require "./helpers/MockDocUpdaterServer" FixturesManager = require "./helpers/FixturesManager" +logger = require("logger-sharelatex") async = require "async" @@ -13,9 +15,13 @@ describe "leaveDoc", -> @lines = ["test", "doc", "lines"] @version = 42 @ops = ["mock", "doc", "ops"] + sinon.spy(logger, "error") + + after -> + logger.error.restore() # remove the spy describe "when joined to a doc", -> - before (done) -> + beforeEach (done) -> async.series [ (cb) => FixturesManager.setUpProject { @@ -39,7 +45,7 @@ describe "leaveDoc", -> ], done describe "then leaving the doc", -> - before (done) -> + beforeEach (done) -> @client.emit "leaveDoc", @doc_id, (error) -> throw error if error? done() @@ -48,3 +54,19 @@ describe "leaveDoc", -> RealTimeClient.getConnectedClient @client.socket.sessionid, (error, client) => expect(@doc_id in client.rooms).to.equal false done() + + describe "when sending a leaveDoc request before the previous joinDoc request has completed", -> + beforeEach (done) -> + @client.emit "leaveDoc", @doc_id, () -> + @client.emit "joinDoc", @doc_id, () -> + @client.emit "leaveDoc", @doc_id, (error) -> + throw error if error? + done() + + it "should not trigger an error", -> + sinon.assert.neverCalledWith(logger.error, sinon.match.any, "not subscribed - shouldn't happen") + + it "should have left the doc room", (done) -> + RealTimeClient.getConnectedClient @client.socket.sessionid, (error, client) => + expect(@doc_id in client.rooms).to.equal false + done() From 478a727c615dd61577aac27a40b5c4bf2b4d42b7 Mon Sep 17 00:00:00 2001 From: Brian Gough Date: Mon, 29 Jul 2019 15:19:08 +0100 Subject: [PATCH 268/491] ignore spurious requests to leave other docs --- .../real-time/app/coffee/RoomManager.coffee | 15 ++++++- .../acceptance/coffee/LeaveDocTests.coffee | 12 ++++++ .../test/unit/coffee/RoomManagerTests.coffee | 41 ++++++++++++++++++- 3 files changed, 66 insertions(+), 2 deletions(-) diff --git a/services/real-time/app/coffee/RoomManager.coffee b/services/real-time/app/coffee/RoomManager.coffee index 0d8a8aaf06..27d69ca379 100644 --- a/services/real-time/app/coffee/RoomManager.coffee +++ b/services/real-time/app/coffee/RoomManager.coffee @@ -32,7 +32,9 @@ module.exports = RoomManager = # channel subscriptions... but it will be safer if we leave them # explicitly, and then socket.io will just regard this as a client that # has not joined any rooms and do a final disconnection. - for id in @_roomsClientIsIn(client) + roomsToLeave = @_roomsClientIsIn(client) + logger.log {client: client.id, roomsToLeave: roomsToLeave}, "client leaving project" + for id in roomsToLeave entity = IdMap.get(id) @leaveEntity client, entity, id @@ -66,6 +68,12 @@ module.exports = RoomManager = callback() leaveEntity: (client, entity, id) -> + # Ignore any requests to leave when the client is not actually in the + # room. This can happen if the client sends spurious leaveDoc requests + # for old docs after a reconnection. + if !@_clientAlreadyInRoom(client, id) + logger.warn {client: client.id, entity, id}, "ignoring request from client to leave room it is not in" + return client.leave id afterCount = @_clientsInRoom(client, id) logger.log {client: client.id, entity, id, afterCount}, "client left room" @@ -93,3 +101,8 @@ module.exports = RoomManager = [prefix, room] = fullRoomPath.split('/', 2) room return roomList + + _clientAlreadyInRoom: (client, room) -> + nsp = client.namespace.name + name = (nsp + '/') + room; + return client.manager.roomClients?[client.id]?[name] \ No newline at end of file diff --git a/services/real-time/test/acceptance/coffee/LeaveDocTests.coffee b/services/real-time/test/acceptance/coffee/LeaveDocTests.coffee index e4875fcbe5..e68b111c64 100644 --- a/services/real-time/test/acceptance/coffee/LeaveDocTests.coffee +++ b/services/real-time/test/acceptance/coffee/LeaveDocTests.coffee @@ -16,9 +16,12 @@ describe "leaveDoc", -> @version = 42 @ops = ["mock", "doc", "ops"] sinon.spy(logger, "error") + sinon.spy(logger, "warn") + @other_doc_id = FixturesManager.getRandomId() after -> logger.error.restore() # remove the spy + logger.warn.restore() describe "when joined to a doc", -> beforeEach (done) -> @@ -70,3 +73,12 @@ describe "leaveDoc", -> RealTimeClient.getConnectedClient @client.socket.sessionid, (error, client) => expect(@doc_id in client.rooms).to.equal false done() + + describe "when sending a leaveDoc for a room the client has not joined ", -> + beforeEach (done) -> + @client.emit "leaveDoc", @other_doc_id, (error) -> + throw error if error? + done() + + it "should trigger a warning only", -> + sinon.assert.calledWith(logger.warn, sinon.match.any, "ignoring request from client to leave room it is not in") \ No newline at end of file diff --git a/services/real-time/test/unit/coffee/RoomManagerTests.coffee b/services/real-time/test/unit/coffee/RoomManagerTests.coffee index ee46a3ef04..fc8375ae54 100644 --- a/services/real-time/test/unit/coffee/RoomManagerTests.coffee +++ b/services/real-time/test/unit/coffee/RoomManagerTests.coffee @@ -12,9 +12,10 @@ describe 'RoomManager', -> @client = {namespace: {name: ''}, id: "first-client"} @RoomManager = SandboxedModule.require modulePath, requires: "settings-sharelatex": @settings = {} - "logger-sharelatex": @logger = { log: sinon.stub(), error: sinon.stub() } + "logger-sharelatex": @logger = { log: sinon.stub(), warn: sinon.stub(), error: sinon.stub() } "metrics-sharelatex": @metrics = { gauge: sinon.stub() } @RoomManager._clientsInRoom = sinon.stub() + @RoomManager._clientAlreadyInRoom = sinon.stub() @RoomEvents = @RoomManager.eventSource() sinon.spy(@RoomEvents, 'emit') sinon.spy(@RoomEvents, 'once') @@ -112,6 +113,9 @@ describe 'RoomManager', -> describe "when doc room will be empty after this client has left", -> beforeEach -> + @RoomManager._clientAlreadyInRoom + .withArgs(@client, @doc_id) + .returns(true) @RoomManager._clientsInRoom .withArgs(@client, @doc_id) .onCall(0).returns(0) @@ -128,6 +132,9 @@ describe 'RoomManager', -> describe "when there are other clients in the doc room", -> beforeEach -> + @RoomManager._clientAlreadyInRoom + .withArgs(@client, @doc_id) + .returns(true) @RoomManager._clientsInRoom .withArgs(@client, @doc_id) .onCall(0).returns(123) @@ -140,6 +147,24 @@ describe 'RoomManager', -> it "should not emit any events", -> @RoomEvents.emit.called.should.equal false + describe "when the client is not in the doc room", -> + + beforeEach -> + @RoomManager._clientAlreadyInRoom + .withArgs(@client, @doc_id) + .returns(false) + @RoomManager._clientsInRoom + .withArgs(@client, @doc_id) + .onCall(0).returns(0) + @client.leave = sinon.stub() + @RoomManager.leaveDoc @client, @doc_id + + it "should not leave the room", -> + @client.leave.called.should.equal false + + it "should not emit any events", -> + @RoomEvents.emit.called.should.equal false + describe "leaveProjectAndDocs", -> @@ -167,6 +192,13 @@ describe 'RoomManager', -> .withArgs(@client, @project_id) .onCall(0).returns(0) .onCall(1).returns(0) + @RoomManager._clientAlreadyInRoom + .withArgs(@client, @doc_id) + .returns(true) + .withArgs(@client, @other_doc_id) + .returns(true) + .withArgs(@client, @project_id) + .returns(true) @RoomEvents.on 'project-active', (id) => setTimeout () => @RoomEvents.emit "project-subscribed-#{id}" @@ -212,6 +244,13 @@ describe 'RoomManager', -> .withArgs(@client, @project_id) .onFirstCall().returns(123) .onSecondCall().returns(122) + @RoomManager._clientAlreadyInRoom + .withArgs(@client, @doc_id) + .returns(true) + .withArgs(@client, @other_doc_id) + .returns(true) + .withArgs(@client, @project_id) + .returns(true) @RoomManager.leaveProjectAndDocs @client it "should leave all the docs", -> From 2000f478a7a657b81e31d64064b41e18a7e0b5a2 Mon Sep 17 00:00:00 2001 From: Brian Gough Date: Tue, 13 Aug 2019 10:39:52 +0100 Subject: [PATCH 269/491] refresh the client list on demand --- .../app/coffee/ConnectedUsersManager.coffee | 14 +++++++++++++- .../app/coffee/WebsocketController.coffee | 13 ++++++++----- .../app/coffee/WebsocketLoadBalancer.coffee | 6 ++++++ 3 files changed, 27 insertions(+), 6 deletions(-) diff --git a/services/real-time/app/coffee/ConnectedUsersManager.coffee b/services/real-time/app/coffee/ConnectedUsersManager.coffee index fe7d7efb85..085a47dc28 100644 --- a/services/real-time/app/coffee/ConnectedUsersManager.coffee +++ b/services/real-time/app/coffee/ConnectedUsersManager.coffee @@ -10,6 +10,7 @@ ONE_DAY_IN_S = ONE_HOUR_IN_S * 24 FOUR_DAYS_IN_S = ONE_DAY_IN_S * 4 USER_TIMEOUT_IN_S = ONE_HOUR_IN_S / 4 +REFRESH_TIMEOUT_IN_S = 10 # only show clients which have responded to a refresh request in the last 10 seconds module.exports = # Use the same method for when a user connects, and when a user sends a cursor @@ -38,6 +39,16 @@ module.exports = logger.err err:err, project_id:project_id, client_id:client_id, "problem marking user as connected" callback(err) + refreshClient: (project_id, client_id, callback = (err) ->) -> + logger.log project_id:project_id, client_id:client_id, "refreshing connected client" + multi = rclient.multi() + multi.hset Keys.connectedUser({project_id, client_id}), "last_updated_at", Date.now() + multi.expire Keys.connectedUser({project_id, client_id}), USER_TIMEOUT_IN_S + multi.exec (err)-> + if err? + logger.err err:err, project_id:project_id, client_id:client_id, "problem refreshing connected client" + callback(err) + markUserAsDisconnected: (project_id, client_id, callback)-> logger.log project_id:project_id, client_id:client_id, "marking user as disconnected" multi = rclient.multi() @@ -56,6 +67,7 @@ module.exports = else result.connected = true result.client_id = client_id + result.client_age = (Date.now() - parseInt(result.last_updated_at,10)) / 1000 if result.cursorData? try result.cursorData = JSON.parse(result.cursorData) @@ -74,6 +86,6 @@ module.exports = async.series jobs, (err, users = [])-> return callback(err) if err? users = users.filter (user) -> - user?.connected + user?.connected && user?.client_age < REFRESH_TIMEOUT_IN_S callback null, users diff --git a/services/real-time/app/coffee/WebsocketController.coffee b/services/real-time/app/coffee/WebsocketController.coffee index fce505c1bc..8a09f534bb 100644 --- a/services/real-time/app/coffee/WebsocketController.coffee +++ b/services/real-time/app/coffee/WebsocketController.coffee @@ -175,6 +175,7 @@ module.exports = WebsocketController = }, callback) WebsocketLoadBalancer.emitToRoom(project_id, "clientTracking.clientUpdated", cursorData) + CLIENT_REFRESH_DELAY: 1000 getConnectedUsers: (client, callback = (error, users) ->) -> metrics.inc "editor.get-connected-users" Utils.getClientAttributes client, ["project_id", "user_id"], (error, {project_id, user_id}) -> @@ -183,11 +184,13 @@ module.exports = WebsocketController = logger.log {user_id, project_id, client_id: client.id}, "getting connected users" AuthorizationManager.assertClientCanViewProject client, (error) -> return callback(error) if error? - ConnectedUsersManager.getConnectedUsers project_id, (error, users) -> - return callback(error) if error? - callback null, users - logger.log {user_id, project_id, client_id: client.id}, "got connected users" - + WebsocketLoadBalancer.emitToRoom project_id, 'clientTracking.refresh', project_id + setTimeout () -> + ConnectedUsersManager.getConnectedUsers project_id, (error, users) -> + return callback(error) if error? + callback null, users + logger.log {user_id, project_id, client_id: client.id}, "got connected users" + , WebsocketController.CLIENT_REFRESH_DELAY applyOtUpdate: (client, doc_id, update, callback = (error) ->) -> Utils.getClientAttributes client, ["user_id", "project_id"], (error, {user_id, project_id}) -> diff --git a/services/real-time/app/coffee/WebsocketLoadBalancer.coffee b/services/real-time/app/coffee/WebsocketLoadBalancer.coffee index 865249c63e..45a7645005 100644 --- a/services/real-time/app/coffee/WebsocketLoadBalancer.coffee +++ b/services/real-time/app/coffee/WebsocketLoadBalancer.coffee @@ -6,6 +6,7 @@ EventLogger = require "./EventLogger" HealthCheckManager = require "./HealthCheckManager" RoomManager = require "./RoomManager" ChannelManager = require "./ChannelManager" +ConnectedUsersManager = require "./ConnectedUsersManager" module.exports = WebsocketLoadBalancer = rclientPubList: RedisClientManager.createClientList(Settings.redis.pubsub) @@ -54,6 +55,11 @@ module.exports = WebsocketLoadBalancer = return if message.room_id == "all" io.sockets.emit(message.message, message.payload...) + else if message.message is 'clientTracking.refresh' && message.room_id? + clientList = io.sockets.clients(message.room_id) + logger.log {channel:channel, message: message.message, room_id: message.room_id, message_id: message._id, socketIoClients: (client.id for client in clientList)}, "refreshing client list" + for client in clientList + ConnectedUsersManager.refreshClient(message.room_id, client.id) else if message.room_id? if message._id? && Settings.checkEventOrder status = EventLogger.checkEventOrder("editor-events", message._id, message) From 5b54d36b375e82488e63eadee35512fe683c4fd5 Mon Sep 17 00:00:00 2001 From: Brian Gough Date: Tue, 13 Aug 2019 10:41:35 +0100 Subject: [PATCH 270/491] fail readiness check when shutting down --- services/real-time/app.coffee | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/services/real-time/app.coffee b/services/real-time/app.coffee index 7b53730ce6..56db154f83 100644 --- a/services/real-time/app.coffee +++ b/services/real-time/app.coffee @@ -56,7 +56,10 @@ app.get "/", (req, res, next) -> res.send "real-time-sharelatex is alive" app.get "/status", (req, res, next) -> - res.send "real-time-sharelatex is alive" + if shutDownInProgress + res.send 503 # Service unavailable + else + res.send "real-time-sharelatex is alive" app.get "/debug/events", (req, res, next) -> Settings.debugEvents = parseInt(req.query?.count,10) || 20 From 00cca29d9ef23156657631c56994c4055dcdd820 Mon Sep 17 00:00:00 2001 From: Henry Oswald Date: Tue, 13 Aug 2019 11:12:04 +0100 Subject: [PATCH 271/491] add shutdownDrainTimeWindow, drains all connections within time range --- services/real-time/app.coffee | 4 ++-- .../real-time/app/coffee/DrainManager.coffee | 7 ++++++- .../real-time/config/settings.defaults.coffee | 2 ++ .../test/unit/coffee/DrainManagerTests.coffee | 18 ++++++++++++++++++ 4 files changed, 28 insertions(+), 3 deletions(-) diff --git a/services/real-time/app.coffee b/services/real-time/app.coffee index 7b53730ce6..8b3486dc0a 100644 --- a/services/real-time/app.coffee +++ b/services/real-time/app.coffee @@ -113,8 +113,8 @@ shutdownCleanly = (signal) -> forceDrain = -> logger.log {delay_ms:Settings.forceDrainMsDelay}, "starting force drain after timeout" setTimeout ()-> - logger.log "starting drain" - DrainManager.startDrain(io, 4) + logger.log "starting drain over #{Settings.shutdownDrainTimeWindow} mins" + DrainManager.startDrainTimeWindow(io, Settings.shutdownDrainTimeWindow) , Settings.forceDrainMsDelay shutDownInProgress = false diff --git a/services/real-time/app/coffee/DrainManager.coffee b/services/real-time/app/coffee/DrainManager.coffee index da6b331e23..a71ffa8ed9 100644 --- a/services/real-time/app/coffee/DrainManager.coffee +++ b/services/real-time/app/coffee/DrainManager.coffee @@ -1,6 +1,11 @@ logger = require "logger-sharelatex" -module.exports = +module.exports = DrainManager = + + startDrainTimeWindow: (io, minsToDrain)-> + drainPerMin = io.sockets.clients().length / minsToDrain + DrainManager.startDrain(io, drainPerMin / 60) + startDrain: (io, rate) -> # Clear out any old interval clearInterval @interval diff --git a/services/real-time/config/settings.defaults.coffee b/services/real-time/config/settings.defaults.coffee index a3128e84d1..b48de98981 100644 --- a/services/real-time/config/settings.defaults.coffee +++ b/services/real-time/config/settings.defaults.coffee @@ -50,6 +50,8 @@ settings = forceDrainMsDelay: process.env['FORCE_DRAIN_MS_DELAY'] or false + shutdownDrainTimeWindow: process.env['SHUTDOWN_DRAIN_TIME_WINDOW'] or 9 + continualPubsubTraffic: process.env['CONTINUAL_PUBSUB_TRAFFIC'] or false checkEventOrder: process.env['CHECK_EVENT_ORDER'] or false diff --git a/services/real-time/test/unit/coffee/DrainManagerTests.coffee b/services/real-time/test/unit/coffee/DrainManagerTests.coffee index b3cdaf1ca0..cd34aea4c4 100644 --- a/services/real-time/test/unit/coffee/DrainManagerTests.coffee +++ b/services/real-time/test/unit/coffee/DrainManagerTests.coffee @@ -12,6 +12,24 @@ describe "DrainManager", -> sockets: clients: sinon.stub() + describe "startDrainTimeWindow", -> + beforeEach -> + @clients = [] + for i in [0..1619] + @clients[i] = { + id: i + emit: sinon.stub() + } + @io.sockets.clients.returns @clients + @DrainManager.startDrain = sinon.stub() + + it "should set a drain rate fast enough", (done)-> + @DrainManager.startDrainTimeWindow(@io, 9) + console.log(@DrainManager.startDrain.args[0]) + @DrainManager.startDrain.calledWith(@io, 3).should.equal true + done() + + describe "reconnectNClients", -> beforeEach -> @clients = [] From b3e5709b649be38aeef72a1f86ccfb537013a4cf Mon Sep 17 00:00:00 2001 From: Brian Gough Date: Tue, 13 Aug 2019 16:15:30 +0100 Subject: [PATCH 272/491] enforce a minimum drain rate --- services/real-time/app/coffee/DrainManager.coffee | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/services/real-time/app/coffee/DrainManager.coffee b/services/real-time/app/coffee/DrainManager.coffee index a71ffa8ed9..2219291267 100644 --- a/services/real-time/app/coffee/DrainManager.coffee +++ b/services/real-time/app/coffee/DrainManager.coffee @@ -4,7 +4,7 @@ module.exports = DrainManager = startDrainTimeWindow: (io, minsToDrain)-> drainPerMin = io.sockets.clients().length / minsToDrain - DrainManager.startDrain(io, drainPerMin / 60) + DrainManager.startDrain(io, Math.max(drainPerMin / 60, 4)) # enforce minimum drain rate startDrain: (io, rate) -> # Clear out any old interval From 53431953fcd55ba064216d0fb98f93564ebec32a Mon Sep 17 00:00:00 2001 From: Brian Gough Date: Tue, 13 Aug 2019 16:56:48 +0100 Subject: [PATCH 273/491] make shutDownInProgress available via settings --- services/real-time/app.coffee | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/services/real-time/app.coffee b/services/real-time/app.coffee index 56db154f83..5a78588591 100644 --- a/services/real-time/app.coffee +++ b/services/real-time/app.coffee @@ -56,7 +56,7 @@ app.get "/", (req, res, next) -> res.send "real-time-sharelatex is alive" app.get "/status", (req, res, next) -> - if shutDownInProgress + if Settings.shutDownInProgress res.send 503 # Service unavailable else res.send "real-time-sharelatex is alive" @@ -120,17 +120,17 @@ forceDrain = -> DrainManager.startDrain(io, 4) , Settings.forceDrainMsDelay -shutDownInProgress = false +Settings.shutDownInProgress = false if Settings.forceDrainMsDelay? Settings.forceDrainMsDelay = parseInt(Settings.forceDrainMsDelay, 10) logger.log forceDrainMsDelay: Settings.forceDrainMsDelay,"forceDrainMsDelay enabled" for signal in ['SIGINT', 'SIGHUP', 'SIGQUIT', 'SIGUSR1', 'SIGUSR2', 'SIGTERM', 'SIGABRT'] process.on signal, -> - if shutDownInProgress + if Settings.shutDownInProgress logger.log signal: signal, "shutdown already in progress, ignoring signal" return else - shutDownInProgress = true + Settings.shutDownInProgress = true logger.log signal: signal, "received interrupt, cleaning up" shutdownCleanly(signal) forceDrain() From 0708f717fd48e1745eac8420543fd8501ab9f6a5 Mon Sep 17 00:00:00 2001 From: Brian Gough Date: Tue, 13 Aug 2019 16:59:15 +0100 Subject: [PATCH 274/491] reject connections when shutdown in progress send a message to the client to reconnect immediately --- services/real-time/app/coffee/Router.coffee | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/services/real-time/app/coffee/Router.coffee b/services/real-time/app/coffee/Router.coffee index ad74ebffa1..838d8de6f5 100644 --- a/services/real-time/app/coffee/Router.coffee +++ b/services/real-time/app/coffee/Router.coffee @@ -39,6 +39,11 @@ module.exports = Router = app.post "/drain", httpAuth, HttpApiController.startDrain session.on 'connection', (error, client, session) -> + if settings.shutDownInProgress + client.emit("connectionRejected", {message: "retry"}) + client.disconnect() + return + if client? and error?.message?.match(/could not look up session by key/) logger.warn err: error, client: client?, session: session?, "invalid session" # tell the client to reauthenticate if it has an invalid session key From 7db882f3398452ef98c62010e49d76d3c6f1186e Mon Sep 17 00:00:00 2001 From: Brian Gough Date: Tue, 13 Aug 2019 17:26:49 +0100 Subject: [PATCH 275/491] fix unit tests --- services/real-time/test/unit/coffee/DrainManagerTests.coffee | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/services/real-time/test/unit/coffee/DrainManagerTests.coffee b/services/real-time/test/unit/coffee/DrainManagerTests.coffee index cd34aea4c4..88009f02cd 100644 --- a/services/real-time/test/unit/coffee/DrainManagerTests.coffee +++ b/services/real-time/test/unit/coffee/DrainManagerTests.coffee @@ -15,7 +15,7 @@ describe "DrainManager", -> describe "startDrainTimeWindow", -> beforeEach -> @clients = [] - for i in [0..1619] + for i in [0..5399] @clients[i] = { id: i emit: sinon.stub() @@ -25,8 +25,7 @@ describe "DrainManager", -> it "should set a drain rate fast enough", (done)-> @DrainManager.startDrainTimeWindow(@io, 9) - console.log(@DrainManager.startDrain.args[0]) - @DrainManager.startDrain.calledWith(@io, 3).should.equal true + @DrainManager.startDrain.calledWith(@io, 10).should.equal true done() From 20d442120f1727626c50d7d56c62ce79f6fd8fc7 Mon Sep 17 00:00:00 2001 From: Brian Gough Date: Tue, 13 Aug 2019 17:36:53 +0100 Subject: [PATCH 276/491] notify docupdate if the flush is from a shutdown --- services/real-time/app/coffee/DocumentUpdaterManager.coffee | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/services/real-time/app/coffee/DocumentUpdaterManager.coffee b/services/real-time/app/coffee/DocumentUpdaterManager.coffee index 40ef992d8f..9968b78753 100644 --- a/services/real-time/app/coffee/DocumentUpdaterManager.coffee +++ b/services/real-time/app/coffee/DocumentUpdaterManager.coffee @@ -40,7 +40,8 @@ module.exports = DocumentUpdaterManager = logger.log project_id:project_id, "deleting project from document updater" timer = new metrics.Timer("delete.mongo.project") # flush the project in the background when all users have left - url = "#{settings.apis.documentupdater.url}/project/#{project_id}?background=true" + url = "#{settings.apis.documentupdater.url}/project/#{project_id}?background=true" + + (if settings.shutDownInProgress then "&shutdown=true" else "") request.del url, (err, res, body)-> timer.done() if err? From 4a984f533e1db3f1bb927f98ecb399843bf69ee3 Mon Sep 17 00:00:00 2001 From: Henry Oswald Date: Wed, 14 Aug 2019 11:51:25 +0100 Subject: [PATCH 277/491] remove forceDrainMsDelay as soon as a pod is marked as being killed we should start draining --- services/real-time/app.coffee | 17 +++++------------ .../real-time/config/settings.defaults.coffee | 2 -- 2 files changed, 5 insertions(+), 14 deletions(-) diff --git a/services/real-time/app.coffee b/services/real-time/app.coffee index 8b3486dc0a..50f7eb975b 100644 --- a/services/real-time/app.coffee +++ b/services/real-time/app.coffee @@ -110,17 +110,10 @@ shutdownCleanly = (signal) -> shutdownCleanly(signal) , 10000 -forceDrain = -> - logger.log {delay_ms:Settings.forceDrainMsDelay}, "starting force drain after timeout" - setTimeout ()-> - logger.log "starting drain over #{Settings.shutdownDrainTimeWindow} mins" - DrainManager.startDrainTimeWindow(io, Settings.shutdownDrainTimeWindow) - , Settings.forceDrainMsDelay - shutDownInProgress = false -if Settings.forceDrainMsDelay? - Settings.forceDrainMsDelay = parseInt(Settings.forceDrainMsDelay, 10) - logger.log forceDrainMsDelay: Settings.forceDrainMsDelay,"forceDrainMsDelay enabled" +if Settings.shutdownDrainTimeWindow? + Settings.forceDrainMsDelay = parseInt(Settings.shutdownDrainTimeWindow, 10) + logger.log shutdownDrainTimeWindow: Settings.shutdownDrainTimeWindow,"shutdownDrainTimeWindow enabled" for signal in ['SIGINT', 'SIGHUP', 'SIGQUIT', 'SIGUSR1', 'SIGUSR2', 'SIGTERM', 'SIGABRT'] process.on signal, -> if shutDownInProgress @@ -128,9 +121,9 @@ if Settings.forceDrainMsDelay? return else shutDownInProgress = true - logger.log signal: signal, "received interrupt, cleaning up" + logger.log signal: signal, "received interrupt, starting drain over #{Settings.shutdownDrainTimeWindow} mins" + DrainManager.startDrainTimeWindow(io, Settings.shutdownDrainTimeWindow) shutdownCleanly(signal) - forceDrain() diff --git a/services/real-time/config/settings.defaults.coffee b/services/real-time/config/settings.defaults.coffee index b48de98981..6fe0f4bd1e 100644 --- a/services/real-time/config/settings.defaults.coffee +++ b/services/real-time/config/settings.defaults.coffee @@ -48,8 +48,6 @@ settings = max_doc_length: 2 * 1024 * 1024 # 2mb - forceDrainMsDelay: process.env['FORCE_DRAIN_MS_DELAY'] or false - shutdownDrainTimeWindow: process.env['SHUTDOWN_DRAIN_TIME_WINDOW'] or 9 continualPubsubTraffic: process.env['CONTINUAL_PUBSUB_TRAFFIC'] or false From d3171e4e2ea715d2c84badc0f444f1597ac4b951 Mon Sep 17 00:00:00 2001 From: Brian Gough Date: Wed, 14 Aug 2019 13:03:06 +0100 Subject: [PATCH 278/491] remove unwanted argument --- services/real-time/app/coffee/WebsocketController.coffee | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/services/real-time/app/coffee/WebsocketController.coffee b/services/real-time/app/coffee/WebsocketController.coffee index 8a09f534bb..5b03f334d5 100644 --- a/services/real-time/app/coffee/WebsocketController.coffee +++ b/services/real-time/app/coffee/WebsocketController.coffee @@ -184,7 +184,7 @@ module.exports = WebsocketController = logger.log {user_id, project_id, client_id: client.id}, "getting connected users" AuthorizationManager.assertClientCanViewProject client, (error) -> return callback(error) if error? - WebsocketLoadBalancer.emitToRoom project_id, 'clientTracking.refresh', project_id + WebsocketLoadBalancer.emitToRoom project_id, 'clientTracking.refresh' setTimeout () -> ConnectedUsersManager.getConnectedUsers project_id, (error, users) -> return callback(error) if error? From d57b229e170615b2f345ecde1d6a7693b20a7799 Mon Sep 17 00:00:00 2001 From: Brian Gough Date: Wed, 14 Aug 2019 13:03:14 +0100 Subject: [PATCH 279/491] update tests --- .../coffee/ConnectedUsersManagerTests.coffee | 16 ++++++++-------- .../unit/coffee/WebsocketControllerTests.coffee | 14 +++++++++++--- 2 files changed, 19 insertions(+), 11 deletions(-) diff --git a/services/real-time/test/unit/coffee/ConnectedUsersManagerTests.coffee b/services/real-time/test/unit/coffee/ConnectedUsersManagerTests.coffee index 7d83cf1a0d..e88b1f27ad 100644 --- a/services/real-time/test/unit/coffee/ConnectedUsersManagerTests.coffee +++ b/services/real-time/test/unit/coffee/ConnectedUsersManagerTests.coffee @@ -147,18 +147,18 @@ describe "ConnectedUsersManager", -> describe "getConnectedUsers", -> beforeEach -> - @users = ["1234", "5678", "9123"] + @users = ["1234", "5678", "9123", "8234"] @rClient.smembers.callsArgWith(1, null, @users) @ConnectedUsersManager._getConnectedUser = sinon.stub() - @ConnectedUsersManager._getConnectedUser.withArgs(@project_id, @users[0]).callsArgWith(2, null, {connected:true, client_id:@users[0]}) - @ConnectedUsersManager._getConnectedUser.withArgs(@project_id, @users[1]).callsArgWith(2, null, {connected:false, client_id:@users[1]}) - @ConnectedUsersManager._getConnectedUser.withArgs(@project_id, @users[2]).callsArgWith(2, null, {connected:true, client_id:@users[2]}) + @ConnectedUsersManager._getConnectedUser.withArgs(@project_id, @users[0]).callsArgWith(2, null, {connected:true, client_age: 2, client_id:@users[0]}) + @ConnectedUsersManager._getConnectedUser.withArgs(@project_id, @users[1]).callsArgWith(2, null, {connected:false, client_age: 1, client_id:@users[1]}) + @ConnectedUsersManager._getConnectedUser.withArgs(@project_id, @users[2]).callsArgWith(2, null, {connected:true, client_age: 3, client_id:@users[2]}) + @ConnectedUsersManager._getConnectedUser.withArgs(@project_id, @users[3]).callsArgWith(2, null, {connected:true, client_age: 11, client_id:@users[3]}) # connected but old - - it "should only return the users in the list which are still in redis", (done)-> + it "should only return the users in the list which are still in redis and recently updated", (done)-> @ConnectedUsersManager.getConnectedUsers @project_id, (err, users)=> users.length.should.equal 2 - users[0].should.deep.equal {client_id:@users[0], connected:true} - users[1].should.deep.equal {client_id:@users[2], connected:true} + users[0].should.deep.equal {client_id:@users[0], client_age: 2, connected:true} + users[1].should.deep.equal {client_id:@users[2], client_age: 3, connected:true} done() diff --git a/services/real-time/test/unit/coffee/WebsocketControllerTests.coffee b/services/real-time/test/unit/coffee/WebsocketControllerTests.coffee index d0dad108e7..edb0055bf5 100644 --- a/services/real-time/test/unit/coffee/WebsocketControllerTests.coffee +++ b/services/real-time/test/unit/coffee/WebsocketControllerTests.coffee @@ -355,18 +355,26 @@ describe 'WebsocketController', -> beforeEach -> @client.params.project_id = @project_id @users = ["mock", "users"] + @WebsocketLoadBalancer.emitToRoom = sinon.stub() @ConnectedUsersManager.getConnectedUsers = sinon.stub().callsArgWith(1, null, @users) describe "when authorized", -> - beforeEach -> + beforeEach (done) -> @AuthorizationManager.assertClientCanViewProject = sinon.stub().callsArgWith(1, null) - @WebsocketController.getConnectedUsers @client, @callback + @WebsocketController.getConnectedUsers @client, (args...) => + @callback(args...) + done() it "should check that the client is authorized to view the project", -> @AuthorizationManager.assertClientCanViewProject .calledWith(@client) .should.equal true - + + it "should broadcast a request to update the client list", -> + @WebsocketLoadBalancer.emitToRoom + .calledWith(@project_id, "clientTracking.refresh") + .should.equal true + it "should get the connected users for the project", -> @ConnectedUsersManager.getConnectedUsers .calledWith(@project_id) From 8270c14d86a45b53a97f653ae0c851a3b3044259 Mon Sep 17 00:00:00 2001 From: Brian Gough Date: Wed, 14 Aug 2019 15:22:03 +0100 Subject: [PATCH 280/491] add connected client count metric --- services/real-time/app/coffee/Router.coffee | 2 ++ 1 file changed, 2 insertions(+) diff --git a/services/real-time/app/coffee/Router.coffee b/services/real-time/app/coffee/Router.coffee index ad74ebffa1..11170d692d 100644 --- a/services/real-time/app/coffee/Router.coffee +++ b/services/real-time/app/coffee/Router.coffee @@ -56,6 +56,7 @@ module.exports = Router = client.emit("connectionAccepted") metrics.inc('socket-io.connection') + metrics.gauge('socket-io.clients', io.sockets.clients()?.length) logger.log session: session, client_id: client.id, "client connected" @@ -77,6 +78,7 @@ module.exports = Router = client.on "disconnect", () -> metrics.inc('socket-io.disconnect') + metrics.gauge('socket-io.clients', io.sockets.clients()?.length) WebsocketController.leaveProject io, client, (err) -> if err? Router._handleError null, err, client, "leaveProject" From f13e66b453dab6de7efdf9223e73d0508328d297 Mon Sep 17 00:00:00 2001 From: Brian Gough Date: Wed, 14 Aug 2019 15:34:23 +0100 Subject: [PATCH 281/491] fix client count so that result is zero when all clients have left --- services/real-time/app/coffee/Router.coffee | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/services/real-time/app/coffee/Router.coffee b/services/real-time/app/coffee/Router.coffee index 11170d692d..a7ed73c981 100644 --- a/services/real-time/app/coffee/Router.coffee +++ b/services/real-time/app/coffee/Router.coffee @@ -78,7 +78,7 @@ module.exports = Router = client.on "disconnect", () -> metrics.inc('socket-io.disconnect') - metrics.gauge('socket-io.clients', io.sockets.clients()?.length) + metrics.gauge('socket-io.clients', io.sockets.clients()?.length - 1) WebsocketController.leaveProject io, client, (err) -> if err? Router._handleError null, err, client, "leaveProject" From 78629610d5e385e0991480ccab645c31bee903d7 Mon Sep 17 00:00:00 2001 From: Henry Oswald Date: Wed, 14 Aug 2019 15:38:02 +0100 Subject: [PATCH 282/491] add health check endpoint and http route logger --- services/real-time/app.coffee | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/services/real-time/app.coffee b/services/real-time/app.coffee index 7b53730ce6..67e25a704c 100644 --- a/services/real-time/app.coffee +++ b/services/real-time/app.coffee @@ -25,10 +25,9 @@ HealthCheckManager = require("./app/js/HealthCheckManager") # work around frame handler bug in socket.io v0.9.16 require("./socket.io.patch.js") - # Set up socket.io server app = express() -Metrics.injectMetricsRoute(app) + server = require('http').createServer(app) io = require('socket.io').listen(server) @@ -38,6 +37,9 @@ cookieParser = CookieParser(Settings.security.sessionSecret) sessionSockets = new SessionSockets(io, sessionStore, cookieParser, Settings.cookieName) +Metrics.injectMetricsRoute(app) +app.use(Metrics.http.monitor(logger)) + io.configure -> io.enable('browser client minification') io.enable('browser client etag') @@ -65,7 +67,7 @@ app.get "/debug/events", (req, res, next) -> rclient = require("redis-sharelatex").createClient(Settings.redis.realtime) -app.get "/health_check/redis", (req, res, next) -> +healthCheck = (req, res, next)-> rclient.healthCheck (error) -> if error? logger.err {err: error}, "failed redis health check" @@ -77,7 +79,11 @@ app.get "/health_check/redis", (req, res, next) -> else res.sendStatus 200 -Metrics.injectMetricsRoute(app) +app.get "/health_check", healthCheck + +app.get "/health_check/redis", healthCheck + + Router = require "./app/js/Router" Router.configure(app, io, sessionSockets) From b0f0fb64ac50049b078bc7602acefce9f29afc26 Mon Sep 17 00:00:00 2001 From: Brian Gough Date: Thu, 15 Aug 2019 09:48:42 +0100 Subject: [PATCH 283/491] clean up unused variable, convert setting to number --- services/real-time/app.coffee | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/services/real-time/app.coffee b/services/real-time/app.coffee index 069b4d6837..61db94ca94 100644 --- a/services/real-time/app.coffee +++ b/services/real-time/app.coffee @@ -121,8 +121,8 @@ shutdownCleanly = (signal) -> Settings.shutDownInProgress = false if Settings.shutdownDrainTimeWindow? - Settings.forceDrainMsDelay = parseInt(Settings.shutdownDrainTimeWindow, 10) - logger.log shutdownDrainTimeWindow: Settings.shutdownDrainTimeWindow,"shutdownDrainTimeWindow enabled" + shutdownDrainTimeWindow = parseInt(Settings.shutdownDrainTimeWindow, 10) + logger.log shutdownDrainTimeWindow: shutdownDrainTimeWindow,"shutdownDrainTimeWindow enabled" for signal in ['SIGINT', 'SIGHUP', 'SIGQUIT', 'SIGUSR1', 'SIGUSR2', 'SIGTERM', 'SIGABRT'] process.on signal, -> if Settings.shutDownInProgress @@ -130,8 +130,8 @@ if Settings.shutdownDrainTimeWindow? return else Settings.shutDownInProgress = true - logger.log signal: signal, "received interrupt, starting drain over #{Settings.shutdownDrainTimeWindow} mins" - DrainManager.startDrainTimeWindow(io, Settings.shutdownDrainTimeWindow) + logger.log signal: signal, "received interrupt, starting drain over #{shutdownDrainTimeWindow} mins" + DrainManager.startDrainTimeWindow(io, shutdownDrainTimeWindow) shutdownCleanly(signal) From 38ed780d80427dcabb6a2a8f823c4bf75106faea Mon Sep 17 00:00:00 2001 From: Henry Oswald Date: Thu, 15 Aug 2019 14:41:22 +0100 Subject: [PATCH 284/491] add log line to draining --- services/real-time/app.coffee | 2 +- services/real-time/app/coffee/DrainManager.coffee | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/services/real-time/app.coffee b/services/real-time/app.coffee index 61db94ca94..708e5f467c 100644 --- a/services/real-time/app.coffee +++ b/services/real-time/app.coffee @@ -130,7 +130,7 @@ if Settings.shutdownDrainTimeWindow? return else Settings.shutDownInProgress = true - logger.log signal: signal, "received interrupt, starting drain over #{shutdownDrainTimeWindow} mins" + logger.warn signal: signal, "received interrupt, starting drain over #{shutdownDrainTimeWindow} mins" DrainManager.startDrainTimeWindow(io, shutdownDrainTimeWindow) shutdownCleanly(signal) diff --git a/services/real-time/app/coffee/DrainManager.coffee b/services/real-time/app/coffee/DrainManager.coffee index 2219291267..ad142ccc47 100644 --- a/services/real-time/app/coffee/DrainManager.coffee +++ b/services/real-time/app/coffee/DrainManager.coffee @@ -9,6 +9,7 @@ module.exports = DrainManager = startDrain: (io, rate) -> # Clear out any old interval clearInterval @interval + logger.log rate: rate, "starting drain" if rate == 0 return else if rate < 1 From fe2e7b3065eb758d2c26e3aa0e8fdcf431f6f0dc Mon Sep 17 00:00:00 2001 From: Brian Gough Date: Fri, 16 Aug 2019 10:07:30 +0100 Subject: [PATCH 285/491] minimal fix for undefined connected users --- services/real-time/app/coffee/ConnectedUsersManager.coffee | 2 +- .../test/unit/coffee/ConnectedUsersManagerTests.coffee | 2 +- .../test/unit/coffee/WebsocketLoadBalancerTests.coffee | 1 + 3 files changed, 3 insertions(+), 2 deletions(-) diff --git a/services/real-time/app/coffee/ConnectedUsersManager.coffee b/services/real-time/app/coffee/ConnectedUsersManager.coffee index 085a47dc28..2e6536c9be 100644 --- a/services/real-time/app/coffee/ConnectedUsersManager.coffee +++ b/services/real-time/app/coffee/ConnectedUsersManager.coffee @@ -60,7 +60,7 @@ module.exports = _getConnectedUser: (project_id, client_id, callback)-> rclient.hgetall Keys.connectedUser({project_id, client_id}), (err, result)-> - if !result? or Object.keys(result).length == 0 + if !result? or Object.keys(result).length == 0 or !result.user_id result = connected : false client_id:client_id diff --git a/services/real-time/test/unit/coffee/ConnectedUsersManagerTests.coffee b/services/real-time/test/unit/coffee/ConnectedUsersManagerTests.coffee index e88b1f27ad..6fb3942b64 100644 --- a/services/real-time/test/unit/coffee/ConnectedUsersManagerTests.coffee +++ b/services/real-time/test/unit/coffee/ConnectedUsersManagerTests.coffee @@ -124,7 +124,7 @@ describe "ConnectedUsersManager", -> it "should return a connected user if there is a user object", (done)-> cursorData = JSON.stringify(cursorData:{row:1}) - @rClient.hgetall.callsArgWith(1, null, {connected_at:new Date(), cursorData}) + @rClient.hgetall.callsArgWith(1, null, {connected_at:new Date(), user_id: @user._id, last_updated_at: "#{Date.now()}", cursorData}) @ConnectedUsersManager._getConnectedUser @project_id, @client_id, (err, result)=> result.connected.should.equal true result.client_id.should.equal @client_id diff --git a/services/real-time/test/unit/coffee/WebsocketLoadBalancerTests.coffee b/services/real-time/test/unit/coffee/WebsocketLoadBalancerTests.coffee index c4f4519790..66e950cd0e 100644 --- a/services/real-time/test/unit/coffee/WebsocketLoadBalancerTests.coffee +++ b/services/real-time/test/unit/coffee/WebsocketLoadBalancerTests.coffee @@ -17,6 +17,7 @@ describe "WebsocketLoadBalancer", -> "./HealthCheckManager": {check: sinon.stub()} "./RoomManager" : @RoomManager = {eventSource: sinon.stub().returns @RoomEvents} "./ChannelManager": @ChannelManager = {publish: sinon.stub()} + "./ConnectedUsersManager": @ConnectedUsersManager = {refreshClient: sinon.stub()} @io = {} @WebsocketLoadBalancer.rclientPubList = [{publish: sinon.stub()}] @WebsocketLoadBalancer.rclientSubList = [{ From 21e294c6ebe811dcf3f4c66943f5a1c5a10462f6 Mon Sep 17 00:00:00 2001 From: Simon Detheridge Date: Sat, 31 Aug 2019 14:04:36 +0100 Subject: [PATCH 286/491] Generate retryable error when hitting rate limits in web --- services/real-time/app/coffee/Errors.coffee | 10 ++++++++++ services/real-time/app/coffee/Router.coffee | 3 +++ .../real-time/app/coffee/WebApiManager.coffee | 6 +++++- .../acceptance/coffee/JoinProjectTests.coffee | 17 +++++++++++++++++ .../coffee/helpers/MockWebServer.coffee | 15 +++++++++------ .../test/unit/coffee/WebApiManagerTests.coffee | 10 ++++++++++ 6 files changed, 54 insertions(+), 7 deletions(-) create mode 100644 services/real-time/app/coffee/Errors.coffee diff --git a/services/real-time/app/coffee/Errors.coffee b/services/real-time/app/coffee/Errors.coffee new file mode 100644 index 0000000000..d6ef3fd71d --- /dev/null +++ b/services/real-time/app/coffee/Errors.coffee @@ -0,0 +1,10 @@ +CodedError = (message, code) -> + error = new Error(message) + error.name = "CodedError" + error.code = code + error.__proto__ = CodedError.prototype + return error +CodedError.prototype.__proto__ = Error.prototype + +module.exports = Errors = + CodedError: CodedError diff --git a/services/real-time/app/coffee/Router.coffee b/services/real-time/app/coffee/Router.coffee index aaa8b59a18..92a17729b7 100644 --- a/services/real-time/app/coffee/Router.coffee +++ b/services/real-time/app/coffee/Router.coffee @@ -21,6 +21,9 @@ module.exports = Router = attrs[key] = value attrs.client_id = client.id attrs.err = error + if error.name == "CodedError" + logger.warn attrs, error.message, code: error.code + return callback {message: error.message, code: error.code} if error.message in ["not authorized", "doc updater could not load requested ops", "no project_id found on client"] logger.warn attrs, error.message return callback {message: error.message} diff --git a/services/real-time/app/coffee/WebApiManager.coffee b/services/real-time/app/coffee/WebApiManager.coffee index f0e7526764..6836d5f213 100644 --- a/services/real-time/app/coffee/WebApiManager.coffee +++ b/services/real-time/app/coffee/WebApiManager.coffee @@ -1,6 +1,7 @@ request = require "request" settings = require "settings-sharelatex" logger = require "logger-sharelatex" +{ CodedError } = require "./Errors" module.exports = WebApiManager = joinProject: (project_id, user, callback = (error, project, privilegeLevel) ->) -> @@ -24,7 +25,10 @@ module.exports = WebApiManager = return callback(error) if error? if 200 <= response.statusCode < 300 callback null, data?.project, data?.privilegeLevel + else if response.statusCode == 429 + logger.log(project_id, user_id, "rate-limit hit when joining project") + callback(new CodedError("rate-limit hit when joining project", "TooManyRequests")) else err = new Error("non-success status code from web: #{response.statusCode}") - logger.error {err, project_id, user_id}, "error accessing web api" + logger.error {err, project_id, user_id}, "error accessing web api" callback err diff --git a/services/real-time/test/acceptance/coffee/JoinProjectTests.coffee b/services/real-time/test/acceptance/coffee/JoinProjectTests.coffee index 471672f9ac..b3707e0981 100644 --- a/services/real-time/test/acceptance/coffee/JoinProjectTests.coffee +++ b/services/real-time/test/acceptance/coffee/JoinProjectTests.coffee @@ -89,3 +89,20 @@ describe "joinProject", -> RealTimeClient.getConnectedClient @client.socket.sessionid, (error, client) => expect(@project_id in client.rooms).to.equal false done() + + describe "when over rate limit", -> + before (done) -> + async.series [ + (cb) => + @client = RealTimeClient.connect() + @client.on "connectionAccepted", cb + + (cb) => + @client.emit "joinProject", project_id: 'rate-limited', (@error) => + cb() + ], done + + it "should return a TooManyRequests error code", -> + @error.message.should.equal "rate-limit hit when joining project" + @error.code.should.equal "TooManyRequests" + diff --git a/services/real-time/test/acceptance/coffee/helpers/MockWebServer.coffee b/services/real-time/test/acceptance/coffee/helpers/MockWebServer.coffee index 06f52a6b19..7c479c59bb 100644 --- a/services/real-time/test/acceptance/coffee/helpers/MockWebServer.coffee +++ b/services/real-time/test/acceptance/coffee/helpers/MockWebServer.coffee @@ -19,12 +19,15 @@ module.exports = MockWebServer = joinProjectRequest: (req, res, next) -> {project_id} = req.params {user_id} = req.query - MockWebServer.joinProject project_id, user_id, (error, project, privilegeLevel) -> - return next(error) if error? - res.json { - project: project - privilegeLevel: privilegeLevel - } + if project_id == 'rate-limited' + res.status(429).send() + else + MockWebServer.joinProject project_id, user_id, (error, project, privilegeLevel) -> + return next(error) if error? + res.json { + project: project + privilegeLevel: privilegeLevel + } running: false run: (callback = (error) ->) -> diff --git a/services/real-time/test/unit/coffee/WebApiManagerTests.coffee b/services/real-time/test/unit/coffee/WebApiManagerTests.coffee index 453169cd54..278585518d 100644 --- a/services/real-time/test/unit/coffee/WebApiManagerTests.coffee +++ b/services/real-time/test/unit/coffee/WebApiManagerTests.coffee @@ -3,6 +3,7 @@ should = chai.should() sinon = require("sinon") modulePath = "../../../app/js/WebApiManager.js" SandboxedModule = require('sandboxed-module') +{ CodedError } = require('../../../app/js/Errors') describe 'WebApiManager', -> beforeEach -> @@ -61,3 +62,12 @@ describe 'WebApiManager', -> .calledWith(new Error("non-success code from web: 500")) .should.equal true + describe "when the project is over its rate limit", -> + beforeEach -> + @request.post = sinon.stub().callsArgWith(1, null, {statusCode: 429}, null) + @WebApiManager.joinProject @project_id, @user_id, @callback + + it "should call the callback with a TooManyRequests error code", -> + @callback + .calledWith(new CodedError("rate-limit hit when joining project", "TooManyRequests")) + .should.equal true From 6765d033396b303de794e8891ca0dc39b54502ae Mon Sep 17 00:00:00 2001 From: Shane Kilkelly Date: Fri, 4 Oct 2019 10:30:24 +0100 Subject: [PATCH 287/491] Track the `isRestrictedUser` flag on clients Then, don't send new chat messages and new comments to those restricted clients. We do this because we don't want to leak private information (email addresses and names) to "restricted" users, those who have read-only access via a shared token. --- .../real-time/app/coffee/WebApiManager.coffee | 4 +- .../app/coffee/WebsocketController.coffee | 30 ++--- .../app/coffee/WebsocketLoadBalancer.coffee | 29 ++++- .../unit/coffee/WebApiManagerTests.coffee | 19 +-- .../coffee/WebsocketControllerTests.coffee | 110 +++++++++--------- .../coffee/WebsocketLoadBalancerTests.coffee | 62 +++++++++- 6 files changed, 165 insertions(+), 89 deletions(-) diff --git a/services/real-time/app/coffee/WebApiManager.coffee b/services/real-time/app/coffee/WebApiManager.coffee index 6836d5f213..ce622d496f 100644 --- a/services/real-time/app/coffee/WebApiManager.coffee +++ b/services/real-time/app/coffee/WebApiManager.coffee @@ -4,7 +4,7 @@ logger = require "logger-sharelatex" { CodedError } = require "./Errors" module.exports = WebApiManager = - joinProject: (project_id, user, callback = (error, project, privilegeLevel) ->) -> + joinProject: (project_id, user, callback = (error, project, privilegeLevel, isRestrictedUser) ->) -> user_id = user._id logger.log {project_id, user_id}, "sending join project request to web" url = "#{settings.apis.web.url}/project/#{project_id}/join" @@ -24,7 +24,7 @@ module.exports = WebApiManager = }, (error, response, data) -> return callback(error) if error? if 200 <= response.statusCode < 300 - callback null, data?.project, data?.privilegeLevel + callback null, data?.project, data?.privilegeLevel, data?.isRestrictedUser else if response.statusCode == 429 logger.log(project_id, user_id, "rate-limit hit when joining project") callback(new CodedError("rate-limit hit when joining project", "TooManyRequests")) diff --git a/services/real-time/app/coffee/WebsocketController.coffee b/services/real-time/app/coffee/WebsocketController.coffee index 5b03f334d5..bc46ce12e0 100644 --- a/services/real-time/app/coffee/WebsocketController.coffee +++ b/services/real-time/app/coffee/WebsocketController.coffee @@ -13,12 +13,12 @@ module.exports = WebsocketController = # it will force a full refresh of the page. Useful for non-backwards # compatible protocol changes. Use only in extreme need. PROTOCOL_VERSION: 2 - + joinProject: (client, user, project_id, callback = (error, project, privilegeLevel, protocolVersion) ->) -> user_id = user?._id logger.log {user_id, project_id, client_id: client.id}, "user joining project" metrics.inc "editor.join-project" - WebApiManager.joinProject project_id, user, (error, project, privilegeLevel) -> + WebApiManager.joinProject project_id, user, (error, project, privilegeLevel, isRestrictedUser) -> return callback(error) if error? if !privilegeLevel or privilegeLevel == "" @@ -26,7 +26,6 @@ module.exports = WebsocketController = logger.warn {err, project_id, user_id, client_id: client.id}, "user is not authorized to join project" return callback(err) - client.set("privilege_level", privilegeLevel) client.set("user_id", user_id) client.set("project_id", project_id) @@ -37,18 +36,19 @@ module.exports = WebsocketController = client.set("connected_time", new Date()) client.set("signup_date", user?.signUpDate) client.set("login_count", user?.loginCount) - + client.set("is_restricted_user", !!(isRestrictedUser)) + RoomManager.joinProject client, project_id, (err) -> logger.log {user_id, project_id, client_id: client.id}, "user joined project" callback null, project, privilegeLevel, WebsocketController.PROTOCOL_VERSION - + # No need to block for setting the user as connected in the cursor tracking ConnectedUsersManager.updateUserPosition project_id, client.id, user, null, () -> - + # We want to flush a project if there are no more (local) connected clients # but we need to wait for the triggering client to disconnect. How long we wait # is determined by FLUSH_IF_EMPTY_DELAY. - FLUSH_IF_EMPTY_DELAY: 500 #ms + FLUSH_IF_EMPTY_DELAY: 500 #ms leaveProject: (io, client, callback = (error) ->) -> metrics.inc "editor.leave-project" Utils.getClientAttributes client, ["project_id", "user_id"], (error, {project_id, user_id}) -> @@ -56,7 +56,7 @@ module.exports = WebsocketController = logger.log {project_id, user_id, client_id: client.id}, "client leaving project" WebsocketLoadBalancer.emitToRoom project_id, "clientTracking.clientDisconnected", client.id - + # bail out if the client had not managed to authenticate or join # the project. Prevents downstream errors in docupdater from # flushProjectToMongoAndDelete with null project_id. @@ -66,12 +66,12 @@ module.exports = WebsocketController = if not project_id? logger.log {user_id: user_id, client_id: client.id}, "client leaving, not in project" return callback() - + # We can do this in the background ConnectedUsersManager.markUserAsDisconnected project_id, client.id, (err) -> if err? logger.error {err, project_id, user_id, client_id: client.id}, "error marking client as disconnected" - + RoomManager.leaveProjectAndDocs(client) setTimeout () -> remainingClients = io.sockets.clients(project_id) @@ -82,14 +82,14 @@ module.exports = WebsocketController = logger.error {err, project_id, user_id, client_id: client.id}, "error flushing to doc updater after leaving project" callback() , WebsocketController.FLUSH_IF_EMPTY_DELAY - + joinDoc: (client, doc_id, fromVersion = -1, options, callback = (error, doclines, version, ops, ranges) ->) -> metrics.inc "editor.join-doc" Utils.getClientAttributes client, ["project_id", "user_id"], (error, {project_id, user_id}) -> return callback(error) if error? return callback(new Error("no project_id found on client")) if !project_id? logger.log {user_id, project_id, doc_id, fromVersion, client_id: client.id}, "client joining doc" - + AuthorizationManager.assertClientCanViewProject client, (error) -> return callback(error) if error? # ensure the per-doc applied-ops channel is subscribed before sending the @@ -124,7 +124,7 @@ module.exports = WebsocketController = AuthorizationManager.addAccessToDoc client, doc_id logger.log {user_id, project_id, doc_id, fromVersion, client_id: client.id}, "client joined doc" callback null, escapedLines, version, ops, ranges - + leaveDoc: (client, doc_id, callback = (error) ->) -> metrics.inc "editor.leave-doc" Utils.getClientAttributes client, ["project_id", "user_id"], (error, {project_id, user_id}) -> @@ -227,9 +227,9 @@ module.exports = WebsocketController = return callback(error) else return callback(null) - + _isCommentUpdate: (update) -> for op in update.op if !op.c? return false - return true \ No newline at end of file + return true diff --git a/services/real-time/app/coffee/WebsocketLoadBalancer.coffee b/services/real-time/app/coffee/WebsocketLoadBalancer.coffee index 45a7645005..21e6235505 100644 --- a/services/real-time/app/coffee/WebsocketLoadBalancer.coffee +++ b/services/real-time/app/coffee/WebsocketLoadBalancer.coffee @@ -7,6 +7,8 @@ HealthCheckManager = require "./HealthCheckManager" RoomManager = require "./RoomManager" ChannelManager = require "./ChannelManager" ConnectedUsersManager = require "./ConnectedUsersManager" +Utils = require './Utils' +Async = require 'async' module.exports = WebsocketLoadBalancer = rclientPubList: RedisClientManager.createClientList(Settings.redis.pubsub) @@ -58,7 +60,7 @@ module.exports = WebsocketLoadBalancer = else if message.message is 'clientTracking.refresh' && message.room_id? clientList = io.sockets.clients(message.room_id) logger.log {channel:channel, message: message.message, room_id: message.room_id, message_id: message._id, socketIoClients: (client.id for client in clientList)}, "refreshing client list" - for client in clientList + for client in clientList ConnectedUsersManager.refreshClient(message.room_id, client.id) else if message.room_id? if message._id? && Settings.checkEventOrder @@ -69,12 +71,27 @@ module.exports = WebsocketLoadBalancer = clientList = io.sockets.clients(message.room_id) # avoid unnecessary work if no clients are connected return if clientList.length is 0 - logger.log {channel:channel, message: message.message, room_id: message.room_id, message_id: message._id, socketIoClients: (client.id for client in clientList)}, "distributing event to clients" + logger.log { + channel: channel, + message: message.message, + room_id: message.room_id, + message_id: message._id, + socketIoClients: (client.id for client in clientList) + }, "distributing event to clients" seen = {} - for client in clientList when not seen[client.id] - seen[client.id] = true - client.emit(message.message, message.payload...) + # Send the messages to clients async, don't wait for them all to finish + Async.eachSeries clientList + , (client, cb) -> + Utils.getClientAttributes client, ['is_restricted_user'], (err, {is_restricted_user}) -> + return cb(err) if err? + if !seen[client.id] + seen[client.id] = true + if !(is_restricted_user && message.message in ['new-chat-message', 'new-comment']) + client.emit(message.message, message.payload...) + cb() + , (err) -> + if err? + logger.err {err, message}, "Error sending message to clients" else if message.health_check? logger.debug {message}, "got health check message in editor events channel" HealthCheckManager.check channel, message.key - diff --git a/services/real-time/test/unit/coffee/WebApiManagerTests.coffee b/services/real-time/test/unit/coffee/WebApiManagerTests.coffee index 278585518d..872666d19b 100644 --- a/services/real-time/test/unit/coffee/WebApiManagerTests.coffee +++ b/services/real-time/test/unit/coffee/WebApiManagerTests.coffee @@ -20,17 +20,18 @@ describe 'WebApiManager', -> user: "username" pass: "password" "logger-sharelatex": @logger = { log: sinon.stub(), error: sinon.stub() } - + describe "joinProject", -> describe "successfully", -> beforeEach -> @response = { project: { name: "Test project" } - privilegeLevel: "owner" + privilegeLevel: "owner", + isRestrictedUser: true } @request.post = sinon.stub().callsArgWith(1, null, {statusCode: 200}, @response) @WebApiManager.joinProject @project_id, @user, @callback - + it "should send a request to web to join the project", -> @request.post .calledWith({ @@ -46,22 +47,22 @@ describe 'WebApiManager', -> headers: {} }) .should.equal true - - it "should return the project and privilegeLevel", -> + + it "should return the project, privilegeLevel, and restricted flag", -> @callback - .calledWith(null, @response.project, @response.privilegeLevel) + .calledWith(null, @response.project, @response.privilegeLevel, @response.isRestrictedUser) .should.equal true - + describe "with an error from web", -> beforeEach -> @request.post = sinon.stub().callsArgWith(1, null, {statusCode: 500}, null) @WebApiManager.joinProject @project_id, @user_id, @callback - + it "should call the callback with an error", -> @callback .calledWith(new Error("non-success code from web: 500")) .should.equal true - + describe "when the project is over its rate limit", -> beforeEach -> @request.post = sinon.stub().callsArgWith(1, null, {statusCode: 429}, null) diff --git a/services/real-time/test/unit/coffee/WebsocketControllerTests.coffee b/services/real-time/test/unit/coffee/WebsocketControllerTests.coffee index edb0055bf5..ab5a503716 100644 --- a/services/real-time/test/unit/coffee/WebsocketControllerTests.coffee +++ b/services/real-time/test/unit/coffee/WebsocketControllerTests.coffee @@ -37,10 +37,10 @@ describe 'WebsocketController', -> inc: sinon.stub() set: sinon.stub() "./RoomManager": @RoomManager = {} - + afterEach -> tk.reset() - + describe "joinProject", -> describe "when authorised", -> beforeEach -> @@ -53,61 +53,65 @@ describe 'WebsocketController', -> } @privilegeLevel = "owner" @ConnectedUsersManager.updateUserPosition = sinon.stub().callsArg(4) - @WebApiManager.joinProject = sinon.stub().callsArgWith(2, null, @project, @privilegeLevel) + @isRestrictedUser = true + @WebApiManager.joinProject = sinon.stub().callsArgWith(2, null, @project, @privilegeLevel, @isRestrictedUser) @RoomManager.joinProject = sinon.stub().callsArg(2) @WebsocketController.joinProject @client, @user, @project_id, @callback - + it "should load the project from web", -> @WebApiManager.joinProject .calledWith(@project_id, @user) .should.equal true - + it "should join the project room", -> @RoomManager.joinProject.calledWith(@client, @project_id).should.equal true - + it "should set the privilege level on the client", -> @client.set.calledWith("privilege_level", @privilegeLevel).should.equal true - + it "should set the user's id on the client", -> @client.set.calledWith("user_id", @user._id).should.equal true - + it "should set the user's email on the client", -> @client.set.calledWith("email", @user.email).should.equal true - + it "should set the user's first_name on the client", -> @client.set.calledWith("first_name", @user.first_name).should.equal true - + it "should set the user's last_name on the client", -> @client.set.calledWith("last_name", @user.last_name).should.equal true - + it "should set the user's sign up date on the client", -> @client.set.calledWith("signup_date", @user.signUpDate).should.equal true - + it "should set the user's login_count on the client", -> @client.set.calledWith("login_count", @user.loginCount).should.equal true - + it "should set the connected time on the client", -> @client.set.calledWith("connected_time", new Date()).should.equal true - + it "should set the project_id on the client", -> @client.set.calledWith("project_id", @project_id).should.equal true - + it "should set the project owner id on the client", -> @client.set.calledWith("owner_id", @owner_id).should.equal true - + + it "should set the is_restricted_user flag on the client", -> + @client.set.calledWith("is_restricted_user", @isRestrictedUser).should.equal true + it "should call the callback with the project, privilegeLevel and protocolVersion", -> @callback .calledWith(null, @project, @privilegeLevel, @WebsocketController.PROTOCOL_VERSION) .should.equal true - + it "should mark the user as connected in ConnectedUsersManager", -> @ConnectedUsersManager.updateUserPosition .calledWith(@project_id, @client.id, @user, null) .should.equal true - + it "should increment the join-project metric", -> @metrics.inc.calledWith("editor.join-project").should.equal true - + describe "when not authorized", -> beforeEach -> @WebApiManager.joinProject = sinon.stub().callsArgWith(2, null, null, null) @@ -138,30 +142,30 @@ describe 'WebsocketController', -> @client.params.user_id = @user_id @WebsocketController.FLUSH_IF_EMPTY_DELAY = 0 tk.reset() # Allow setTimeout to work. - + describe "when the project is empty", -> beforeEach (done) -> @clientsInRoom = [] @WebsocketController.leaveProject @io, @client, done - + it "should end clientTracking.clientDisconnected to the project room", -> @WebsocketLoadBalancer.emitToRoom .calledWith(@project_id, "clientTracking.clientDisconnected", @client.id) .should.equal true - + it "should mark the user as disconnected", -> @ConnectedUsersManager.markUserAsDisconnected .calledWith(@project_id, @client.id) .should.equal true - + it "should flush the project in the document updater", -> @DocumentUpdaterManager.flushProjectToMongoAndDelete .calledWith(@project_id) .should.equal true - + it "should increment the leave-project metric", -> @metrics.inc.calledWith("editor.leave-project").should.equal true - + it "should track the disconnection in RoomManager", -> @RoomManager.leaveProjectAndDocs .calledWith(@client) @@ -171,7 +175,7 @@ describe 'WebsocketController', -> beforeEach -> @clientsInRoom = ["mock-remaining-client"] @WebsocketController.leaveProject @io, @client - + it "should not flush the project in the document updater", -> @DocumentUpdaterManager.flushProjectToMongoAndDelete .called.should.equal false @@ -232,7 +236,7 @@ describe 'WebsocketController', -> @ops = ["mock", "ops"] @ranges = { "mock": "ranges" } @options = {} - + @client.params.project_id = @project_id @AuthorizationManager.addAccessToDoc = sinon.stub() @AuthorizationManager.assertClientCanViewProject = sinon.stub().callsArgWith(1, null) @@ -275,7 +279,7 @@ describe 'WebsocketController', -> beforeEach -> @fromVersion = 40 @WebsocketController.joinDoc @client, @doc_id, @fromVersion, @options, @callback - + it "should get the document from the DocumentUpdaterManager with fromVersion", -> @DocumentUpdaterManager.getDocument .calledWith(@project_id, @doc_id, @fromVersion) @@ -285,7 +289,7 @@ describe 'WebsocketController', -> beforeEach -> @doc_lines.push ["räksmörgås"] @WebsocketController.joinDoc @client, @doc_id, -1, @options, @callback - + it "should call the callback with the escaped lines", -> escaped_lines = @callback.args[0][1] escaped_word = escaped_lines.pop() @@ -327,44 +331,44 @@ describe 'WebsocketController', -> beforeEach -> @AuthorizationManager.assertClientCanViewProject = sinon.stub().callsArgWith(1, @err = new Error("not authorized")) @WebsocketController.joinDoc @client, @doc_id, -1, @options, @callback - + it "should call the callback with an error", -> @callback.calledWith(@err).should.equal true - + it "should not call the DocumentUpdaterManager", -> @DocumentUpdaterManager.getDocument.called.should.equal false - + describe "leaveDoc", -> beforeEach -> - @doc_id = "doc-id-123" + @doc_id = "doc-id-123" @client.params.project_id = @project_id @RoomManager.leaveDoc = sinon.stub() @WebsocketController.leaveDoc @client, @doc_id, @callback - + it "should remove the client from the doc_id room", -> @RoomManager.leaveDoc .calledWith(@client, @doc_id).should.equal true - + it "should call the callback", -> @callback.called.should.equal true - + it "should increment the leave-doc metric", -> @metrics.inc.calledWith("editor.leave-doc").should.equal true - + describe "getConnectedUsers", -> beforeEach -> @client.params.project_id = @project_id @users = ["mock", "users"] @WebsocketLoadBalancer.emitToRoom = sinon.stub() @ConnectedUsersManager.getConnectedUsers = sinon.stub().callsArgWith(1, null, @users) - + describe "when authorized", -> beforeEach (done) -> @AuthorizationManager.assertClientCanViewProject = sinon.stub().callsArgWith(1, null) @WebsocketController.getConnectedUsers @client, (args...) => @callback(args...) done() - + it "should check that the client is authorized to view the project", -> @AuthorizationManager.assertClientCanViewProject .calledWith(@client) @@ -379,26 +383,26 @@ describe 'WebsocketController', -> @ConnectedUsersManager.getConnectedUsers .calledWith(@project_id) .should.equal true - + it "should return the users", -> @callback.calledWith(null, @users).should.equal true - + it "should increment the get-connected-users metric", -> @metrics.inc.calledWith("editor.get-connected-users").should.equal true - + describe "when not authorized", -> beforeEach -> @AuthorizationManager.assertClientCanViewProject = sinon.stub().callsArgWith(1, @err = new Error("not authorized")) @WebsocketController.getConnectedUsers @client, @callback - + it "should not get the connected users for the project", -> @ConnectedUsersManager.getConnectedUsers .called .should.equal false - + it "should return an error", -> @callback.calledWith(@err).should.equal true - + describe "updateClientPosition", -> beforeEach -> @WebsocketLoadBalancer.emitToRoom = sinon.stub() @@ -422,7 +426,7 @@ describe 'WebsocketController', -> @client.get = (param, callback) => callback null, @clientParams[param] @WebsocketController.updateClientPosition @client, @update - @populatedCursorData = + @populatedCursorData = doc_id: @doc_id, id: @client.id name: "#{@first_name} #{@last_name}" @@ -462,7 +466,7 @@ describe 'WebsocketController', -> @client.get = (param, callback) => callback null, @clientParams[param] @WebsocketController.updateClientPosition @client, @update - @populatedCursorData = + @populatedCursorData = doc_id: @doc_id, id: @client.id name: "#{@first_name}" @@ -502,7 +506,7 @@ describe 'WebsocketController', -> @client.get = (param, callback) => callback null, @clientParams[param] @WebsocketController.updateClientPosition @client, @update - @populatedCursorData = + @populatedCursorData = doc_id: @doc_id, id: @client.id name: "#{@last_name}" @@ -603,7 +607,7 @@ describe 'WebsocketController', -> it "should call the callback", -> @callback.called.should.equal true - + it "should increment the doc updates", -> @metrics.inc.calledWith("editor.doc-update").should.equal true @@ -621,7 +625,7 @@ describe 'WebsocketController', -> it "should call the callback with the error", -> @callback.calledWith(@error).should.equal true - + describe "when not authorized", -> beforeEach -> @client.disconnect = sinon.stub() @@ -645,7 +649,7 @@ describe 'WebsocketController', -> @comment_update = { op: [{c: "bar", p: 132}] } @AuthorizationManager.assertClientCanEditProjectAndDoc = sinon.stub() @AuthorizationManager.assertClientCanViewProjectAndDoc = sinon.stub() - + describe "with a read-write client", -> it "should return successfully", (done) -> @AuthorizationManager.assertClientCanEditProjectAndDoc.yields(null) @@ -660,7 +664,7 @@ describe 'WebsocketController', -> @WebsocketController._assertClientCanApplyUpdate @client, @doc_id, @edit_update, (error) -> expect(error.message).to.equal "not authorized" done() - + describe "with a read-only client and a comment op", -> it "should return successfully", (done) -> @AuthorizationManager.assertClientCanEditProjectAndDoc.yields(new Error("not authorized")) @@ -668,7 +672,7 @@ describe 'WebsocketController', -> @WebsocketController._assertClientCanApplyUpdate @client, @doc_id, @comment_update, (error) -> expect(error).to.be.null done() - + describe "with a totally unauthorized client", -> it "should return an error", (done) -> @AuthorizationManager.assertClientCanEditProjectAndDoc.yields(new Error("not authorized")) diff --git a/services/real-time/test/unit/coffee/WebsocketLoadBalancerTests.coffee b/services/real-time/test/unit/coffee/WebsocketLoadBalancerTests.coffee index 66e950cd0e..a1906ab72c 100644 --- a/services/real-time/test/unit/coffee/WebsocketLoadBalancerTests.coffee +++ b/services/real-time/test/unit/coffee/WebsocketLoadBalancerTests.coffee @@ -8,7 +8,7 @@ describe "WebsocketLoadBalancer", -> @rclient = {} @RoomEvents = {on: sinon.stub()} @WebsocketLoadBalancer = SandboxedModule.require modulePath, requires: - "./RedisClientManager": + "./RedisClientManager": createClientList: () => [] "logger-sharelatex": @logger = { log: sinon.stub(), error: sinon.stub() } "./SafeJsonParse": @SafeJsonParse = @@ -18,6 +18,12 @@ describe "WebsocketLoadBalancer", -> "./RoomManager" : @RoomManager = {eventSource: sinon.stub().returns @RoomEvents} "./ChannelManager": @ChannelManager = {publish: sinon.stub()} "./ConnectedUsersManager": @ConnectedUsersManager = {refreshClient: sinon.stub()} + "./Utils": @Utils = { + getClientAttributes: sinon.spy( + (client, _attrs, callback) -> + callback(null, {is_restricted_user: !!client.__isRestricted}) + ) + } @io = {} @WebsocketLoadBalancer.rclientPubList = [{publish: sinon.stub()}] @WebsocketLoadBalancer.rclientSubList = [{ @@ -51,7 +57,7 @@ describe "WebsocketLoadBalancer", -> @WebsocketLoadBalancer.emitToRoom .calledWith("all", @message, @payload...) .should.equal true - + describe "listenForEditorEvents", -> beforeEach -> @WebsocketLoadBalancer._processEditorEvent = sinon.stub() @@ -70,9 +76,10 @@ describe "WebsocketLoadBalancer", -> describe "_processEditorEvent", -> describe "with bad JSON", -> beforeEach -> + @isRestrictedUser = false @SafeJsonParse.parse = sinon.stub().callsArgWith 1, new Error("oops") @WebsocketLoadBalancer._processEditorEvent(@io, "editor-events", "blah") - + it "should log an error", -> @logger.error.called.should.equal true @@ -98,6 +105,54 @@ describe "WebsocketLoadBalancer", -> @emit2.calledWith(@message, @payload...).should.equal true @emit3.called.should.equal false # duplicate client should be ignored + describe "with a designated room, and restricted clients, not restricted message", -> + beforeEach -> + @io.sockets = + clients: sinon.stub().returns([ + {id: 'client-id-1', emit: @emit1 = sinon.stub()} + {id: 'client-id-2', emit: @emit2 = sinon.stub()} + {id: 'client-id-1', emit: @emit3 = sinon.stub()} # duplicate client + {id: 'client-id-4', emit: @emit4 = sinon.stub(), __isRestricted: true} + ]) + data = JSON.stringify + room_id: @room_id + message: @message + payload: @payload + @WebsocketLoadBalancer._processEditorEvent(@io, "editor-events", data) + + it "should send the message to all (unique) clients in the room", -> + @io.sockets.clients + .calledWith(@room_id) + .should.equal true + @emit1.calledWith(@message, @payload...).should.equal true + @emit2.calledWith(@message, @payload...).should.equal true + @emit3.called.should.equal false # duplicate client should be ignored + @emit4.called.should.equal true # restricted client, but should be called + + describe "with a designated room, and restricted clients, restricted message", -> + beforeEach -> + @io.sockets = + clients: sinon.stub().returns([ + {id: 'client-id-1', emit: @emit1 = sinon.stub()} + {id: 'client-id-2', emit: @emit2 = sinon.stub()} + {id: 'client-id-1', emit: @emit3 = sinon.stub()} # duplicate client + {id: 'client-id-4', emit: @emit4 = sinon.stub(), __isRestricted: true} + ]) + data = JSON.stringify + room_id: @room_id + message: @restrictedMessage = 'new-comment' + payload: @payload + @WebsocketLoadBalancer._processEditorEvent(@io, "editor-events", data) + + it "should send the message to all (unique) clients in the room, who are not restricted", -> + @io.sockets.clients + .calledWith(@room_id) + .should.equal true + @emit1.calledWith(@restrictedMessage, @payload...).should.equal true + @emit2.calledWith(@restrictedMessage, @payload...).should.equal true + @emit3.called.should.equal false # duplicate client should be ignored + @emit4.called.should.equal false # restricted client, should not be called + describe "when emitting to all", -> beforeEach -> @io.sockets = @@ -110,4 +165,3 @@ describe "WebsocketLoadBalancer", -> it "should send the message to all clients", -> @emit.calledWith(@message, @payload...).should.equal true - From df6cd4a0544520b413536d26f63fff0049317dbb Mon Sep 17 00:00:00 2001 From: Shane Kilkelly Date: Fri, 4 Oct 2019 13:41:49 +0100 Subject: [PATCH 288/491] Also block getConnectedUsers for restricted users. Plus refactor to use a pass list instead of a deny list. --- .../app/coffee/WebsocketController.coffee | 5 ++++- .../app/coffee/WebsocketLoadBalancer.coffee | 13 ++++++++++++- .../unit/coffee/WebsocketControllerTests.coffee | 14 ++++++++++++++ .../unit/coffee/WebsocketLoadBalancerTests.coffee | 2 +- 4 files changed, 31 insertions(+), 3 deletions(-) diff --git a/services/real-time/app/coffee/WebsocketController.coffee b/services/real-time/app/coffee/WebsocketController.coffee index bc46ce12e0..d0ca99cc5c 100644 --- a/services/real-time/app/coffee/WebsocketController.coffee +++ b/services/real-time/app/coffee/WebsocketController.coffee @@ -178,8 +178,11 @@ module.exports = WebsocketController = CLIENT_REFRESH_DELAY: 1000 getConnectedUsers: (client, callback = (error, users) ->) -> metrics.inc "editor.get-connected-users" - Utils.getClientAttributes client, ["project_id", "user_id"], (error, {project_id, user_id}) -> + Utils.getClientAttributes client, ["project_id", "user_id", "is_restricted_user"], (error, clientAttributes) -> return callback(error) if error? + {project_id, user_id, is_restricted_user} = clientAttributes + if is_restricted_user + return callback(null, []) return callback(new Error("no project_id found on client")) if !project_id? logger.log {user_id, project_id, client_id: client.id}, "getting connected users" AuthorizationManager.assertClientCanViewProject client, (error) -> diff --git a/services/real-time/app/coffee/WebsocketLoadBalancer.coffee b/services/real-time/app/coffee/WebsocketLoadBalancer.coffee index 21e6235505..ba69d79beb 100644 --- a/services/real-time/app/coffee/WebsocketLoadBalancer.coffee +++ b/services/real-time/app/coffee/WebsocketLoadBalancer.coffee @@ -10,6 +10,17 @@ ConnectedUsersManager = require "./ConnectedUsersManager" Utils = require './Utils' Async = require 'async' +RESTRICTED_USER_MESSAGE_TYPE_PASS_LIST = [ + 'connectionAccepted', + 'otUpdateApplied', + 'otUpdateError', + 'joinDoc', + 'reciveNewDoc', + 'reciveNewFile', + 'reciveNewFolder', + 'removeEntity' +] + module.exports = WebsocketLoadBalancer = rclientPubList: RedisClientManager.createClientList(Settings.redis.pubsub) rclientSubList: RedisClientManager.createClientList(Settings.redis.pubsub) @@ -86,7 +97,7 @@ module.exports = WebsocketLoadBalancer = return cb(err) if err? if !seen[client.id] seen[client.id] = true - if !(is_restricted_user && message.message in ['new-chat-message', 'new-comment']) + if !(is_restricted_user && message.message not in RESTRICTED_USER_MESSAGE_TYPE_PASS_LIST) client.emit(message.message, message.payload...) cb() , (err) -> diff --git a/services/real-time/test/unit/coffee/WebsocketControllerTests.coffee b/services/real-time/test/unit/coffee/WebsocketControllerTests.coffee index ab5a503716..116485384d 100644 --- a/services/real-time/test/unit/coffee/WebsocketControllerTests.coffee +++ b/services/real-time/test/unit/coffee/WebsocketControllerTests.coffee @@ -403,6 +403,20 @@ describe 'WebsocketController', -> it "should return an error", -> @callback.calledWith(@err).should.equal true + describe "when restricted user", -> + beforeEach -> + @client.params.is_restricted_user = true + @AuthorizationManager.assertClientCanViewProject = sinon.stub().callsArgWith(1, null) + @WebsocketController.getConnectedUsers @client, @callback + + it "should return an empty array of users", -> + @callback.calledWith(null, []).should.equal true + + it "should not get the connected users for the project", -> + @ConnectedUsersManager.getConnectedUsers + .called + .should.equal false + describe "updateClientPosition", -> beforeEach -> @WebsocketLoadBalancer.emitToRoom = sinon.stub() diff --git a/services/real-time/test/unit/coffee/WebsocketLoadBalancerTests.coffee b/services/real-time/test/unit/coffee/WebsocketLoadBalancerTests.coffee index a1906ab72c..e6fe1df8c9 100644 --- a/services/real-time/test/unit/coffee/WebsocketLoadBalancerTests.coffee +++ b/services/real-time/test/unit/coffee/WebsocketLoadBalancerTests.coffee @@ -32,7 +32,7 @@ describe "WebsocketLoadBalancer", -> }] @room_id = "room-id" - @message = "message-to-editor" + @message = "otUpdateApplied" @payload = ["argument one", 42] describe "emitToRoom", -> From 85b23d7da741855d6820a6117ccbb858b0df37ae Mon Sep 17 00:00:00 2001 From: Simon Detheridge Date: Thu, 10 Oct 2019 09:49:24 +0100 Subject: [PATCH 289/491] Add maxRetriesPerRequest setting for redis --- services/real-time/config/settings.defaults.coffee | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/services/real-time/config/settings.defaults.coffee b/services/real-time/config/settings.defaults.coffee index 6fe0f4bd1e..7fba6ea0d6 100644 --- a/services/real-time/config/settings.defaults.coffee +++ b/services/real-time/config/settings.defaults.coffee @@ -5,6 +5,7 @@ settings = host: process.env['PUBSUB_REDIS_HOST'] or process.env['REDIS_HOST'] or "localhost" port: process.env['PUBSUB_REDIS_PORT'] or process.env['REDIS_PORT'] or "6379" password: process.env["PUBSUB_REDIS_PASSWORD"] or process.env["REDIS_PASSWORD"] or "" + maxRetriesPerRequest: parseInt(process.env["PUBSUB_REDIS_MAX_RETRIES_PER_REQUEST"] or process.env["REDIS_MAX_RETRIES_PER_REQUEST"] or "20") realtime: host: process.env['REAL_TIME_REDIS_HOST'] or process.env['REDIS_HOST'] or "localhost" @@ -13,6 +14,7 @@ settings = key_schema: clientsInProject: ({project_id}) -> "clients_in_project:{#{project_id}}" connectedUser: ({project_id, client_id})-> "connected_user:{#{project_id}}:#{client_id}" + maxRetriesPerRequest: parseInt(process.env["REAL_TIME_REDIS_MAX_RETRIES_PER_REQUEST"] or process.env["REDIS_MAX_RETRIES_PER_REQUEST"] or "20") documentupdater: host: process.env['DOC_UPDATER_REDIS_HOST'] or process.env['REDIS_HOST'] or "localhost" @@ -20,11 +22,13 @@ settings = password: process.env["DOC_UPDATER_REDIS_PASSWORD"] or process.env["REDIS_PASSWORD"] or "" key_schema: pendingUpdates: ({doc_id}) -> "PendingUpdates:{#{doc_id}}" + maxRetriesPerRequest: parseInt(process.env["DOC_UPDATER_REDIS_MAX_RETRIES_PER_REQUEST"] or process.env["REDIS_MAX_RETRIES_PER_REQUEST"] or "20") websessions: host: process.env['WEB_REDIS_HOST'] or process.env['REDIS_HOST'] or "localhost" port: process.env['WEB_REDIS_PORT'] or process.env['REDIS_PORT'] or "6379" password: process.env["WEB_REDIS_PASSWORD"] or process.env["REDIS_PASSWORD"] or "" + maxRetriesPerRequest: parseInt(process.env["WEB_REDIS_MAX_RETRIES_PER_REQUEST"] or process.env["REDIS_MAX_RETRIES_PER_REQUEST"] or "20") internal: realTime: @@ -60,4 +64,4 @@ settings = dsn: process.env.SENTRY_DSN # console.log settings.redis -module.exports = settings \ No newline at end of file +module.exports = settings From 06aa578bdcbc8041396a58c2d1cc59cf303ae8ea Mon Sep 17 00:00:00 2001 From: Shane Kilkelly Date: Fri, 11 Oct 2019 09:57:16 +0100 Subject: [PATCH 290/491] Make it an error when we get no data from joinProject --- services/real-time/app/coffee/WebApiManager.coffee | 6 +++++- .../test/unit/coffee/WebApiManagerTests.coffee | 10 ++++++++++ 2 files changed, 15 insertions(+), 1 deletion(-) diff --git a/services/real-time/app/coffee/WebApiManager.coffee b/services/real-time/app/coffee/WebApiManager.coffee index ce622d496f..3c0551a815 100644 --- a/services/real-time/app/coffee/WebApiManager.coffee +++ b/services/real-time/app/coffee/WebApiManager.coffee @@ -24,7 +24,11 @@ module.exports = WebApiManager = }, (error, response, data) -> return callback(error) if error? if 200 <= response.statusCode < 300 - callback null, data?.project, data?.privilegeLevel, data?.isRestrictedUser + if !data? || !data?.project? + err = new Error('no data returned from joinProject request') + logger.error {err, project_id, user_id}, "error accessing web api" + return callback(err) + callback null, data.project, data.privilegeLevel, data.isRestrictedUser else if response.statusCode == 429 logger.log(project_id, user_id, "rate-limit hit when joining project") callback(new CodedError("rate-limit hit when joining project", "TooManyRequests")) diff --git a/services/real-time/test/unit/coffee/WebApiManagerTests.coffee b/services/real-time/test/unit/coffee/WebApiManagerTests.coffee index 872666d19b..a87522c0a9 100644 --- a/services/real-time/test/unit/coffee/WebApiManagerTests.coffee +++ b/services/real-time/test/unit/coffee/WebApiManagerTests.coffee @@ -63,6 +63,16 @@ describe 'WebApiManager', -> .calledWith(new Error("non-success code from web: 500")) .should.equal true + describe "with no data from web", -> + beforeEach -> + @request.post = sinon.stub().callsArgWith(1, null, {statusCode: 200}, null) + @WebApiManager.joinProject @project_id, @user_id, @callback + + it "should call the callback with an error", -> + @callback + .calledWith(new Error("no data returned from joinProject request")) + .should.equal true + describe "when the project is over its rate limit", -> beforeEach -> @request.post = sinon.stub().callsArgWith(1, null, {statusCode: 429}, null) From 2cc2be3d9cda293dcedfc85723f4762075b551d9 Mon Sep 17 00:00:00 2001 From: Shane Kilkelly Date: Fri, 11 Oct 2019 10:01:21 +0100 Subject: [PATCH 291/491] send messages to clients with concurrency of 2 --- services/real-time/app/coffee/WebsocketLoadBalancer.coffee | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/services/real-time/app/coffee/WebsocketLoadBalancer.coffee b/services/real-time/app/coffee/WebsocketLoadBalancer.coffee index ba69d79beb..12b7ef812b 100644 --- a/services/real-time/app/coffee/WebsocketLoadBalancer.coffee +++ b/services/real-time/app/coffee/WebsocketLoadBalancer.coffee @@ -91,7 +91,8 @@ module.exports = WebsocketLoadBalancer = }, "distributing event to clients" seen = {} # Send the messages to clients async, don't wait for them all to finish - Async.eachSeries clientList + Async.eachLimit clientList + , 2 , (client, cb) -> Utils.getClientAttributes client, ['is_restricted_user'], (err, {is_restricted_user}) -> return cb(err) if err? From f028148fe257401f7c1a0bc2f9cb2756e6cdc38d Mon Sep 17 00:00:00 2001 From: Brian Gough Date: Mon, 14 Oct 2019 11:10:20 +0100 Subject: [PATCH 292/491] upgrade ioredis to v4.14.1 --- services/real-time/npm-shrinkwrap.json | 28 +++++++++++++------------- services/real-time/package.json | 4 ++-- 2 files changed, 16 insertions(+), 16 deletions(-) diff --git a/services/real-time/npm-shrinkwrap.json b/services/real-time/npm-shrinkwrap.json index a146f50e1f..bc7bba57e5 100644 --- a/services/real-time/npm-shrinkwrap.json +++ b/services/real-time/npm-shrinkwrap.json @@ -373,9 +373,9 @@ "dev": true }, "cluster-key-slot": { - "version": "1.0.12", + "version": "1.1.0", "from": "cluster-key-slot@>=1.0.6 <2.0.0", - "resolved": "https://registry.npmjs.org/cluster-key-slot/-/cluster-key-slot-1.0.12.tgz" + "resolved": "https://registry.npmjs.org/cluster-key-slot/-/cluster-key-slot-1.1.0.tgz" }, "coffee-script": { "version": "1.6.0", @@ -884,9 +884,9 @@ "resolved": "https://registry.npmjs.org/inherits/-/inherits-2.0.3.tgz" }, "ioredis": { - "version": "4.11.1", - "from": "ioredis@>=4.11.1 <4.12.0", - "resolved": "https://registry.npmjs.org/ioredis/-/ioredis-4.11.1.tgz", + "version": "4.14.1", + "from": "ioredis@>=4.14.1 <4.15.0", + "resolved": "https://registry.npmjs.org/ioredis/-/ioredis-4.14.1.tgz", "dependencies": { "debug": { "version": "4.1.1", @@ -1014,9 +1014,9 @@ "resolved": "https://registry.npmjs.org/jws/-/jws-3.2.2.tgz" }, "lodash": { - "version": "4.17.11", - "from": "lodash@>=4.17.10 <5.0.0", - "resolved": "https://registry.npmjs.org/lodash/-/lodash-4.17.11.tgz" + "version": "4.17.15", + "from": "lodash@>=4.17.14 <5.0.0", + "resolved": "https://registry.npmjs.org/lodash/-/lodash-4.17.15.tgz" }, "lodash.defaults": { "version": "4.2.0", @@ -1445,14 +1445,14 @@ } }, "redis-sharelatex": { - "version": "1.0.9", - "from": "redis-sharelatex@1.0.9", - "resolved": "https://registry.npmjs.org/redis-sharelatex/-/redis-sharelatex-1.0.9.tgz", + "version": "1.0.11", + "from": "redis-sharelatex@1.0.11", + "resolved": "https://registry.npmjs.org/redis-sharelatex/-/redis-sharelatex-1.0.11.tgz", "dependencies": { "async": { - "version": "2.6.2", - "from": "async@https://registry.npmjs.org/async/-/async-2.6.2.tgz", - "resolved": "https://registry.npmjs.org/async/-/async-2.6.2.tgz" + "version": "2.6.3", + "from": "async@>=2.5.0 <3.0.0", + "resolved": "https://registry.npmjs.org/async/-/async-2.6.3.tgz" }, "coffee-script": { "version": "1.8.0", diff --git a/services/real-time/package.json b/services/real-time/package.json index dd2d6f0198..ba94412efe 100644 --- a/services/real-time/package.json +++ b/services/real-time/package.json @@ -30,7 +30,7 @@ "express-session": "^1.9.1", "logger-sharelatex": "^1.7.0", "metrics-sharelatex": "^2.2.0", - "redis-sharelatex": "^1.0.9", + "redis-sharelatex": "^1.0.11", "request": "^2.88.0", "session.socket.io": "^0.1.6", "settings-sharelatex": "^1.1.0", @@ -47,4 +47,4 @@ "uid-safe": "^1.0.1", "timekeeper": "0.0.4" } -} \ No newline at end of file +} From 7543f2fcbd39b4b7c2756cb61445d077a6bdc511 Mon Sep 17 00:00:00 2001 From: Simon Detheridge Date: Thu, 17 Oct 2019 12:45:56 +0100 Subject: [PATCH 293/491] Catch errors from socket.io and attempt graceful cleanup --- services/real-time/app.coffee | 28 +++++++++++++------ .../real-time/app/coffee/DrainManager.coffee | 2 +- services/real-time/app/coffee/Router.coffee | 6 ++++ .../real-time/config/settings.defaults.coffee | 4 +++ 4 files changed, 30 insertions(+), 10 deletions(-) diff --git a/services/real-time/app.coffee b/services/real-time/app.coffee index 708e5f467c..c2f9e7fa9c 100644 --- a/services/real-time/app.coffee +++ b/services/real-time/app.coffee @@ -119,22 +119,32 @@ shutdownCleanly = (signal) -> shutdownCleanly(signal) , 10000 +drainAndShutdown = (signal) -> + if Settings.shutDownInProgress + logger.log signal: signal, "shutdown already in progress, ignoring signal" + return + else + Settings.shutDownInProgress = true + logger.warn signal: signal, "received interrupt, starting drain over #{shutdownDrainTimeWindow} mins" + DrainManager.startDrainTimeWindow(io, shutdownDrainTimeWindow) + shutdownCleanly(signal) + + Settings.shutDownInProgress = false if Settings.shutdownDrainTimeWindow? shutdownDrainTimeWindow = parseInt(Settings.shutdownDrainTimeWindow, 10) logger.log shutdownDrainTimeWindow: shutdownDrainTimeWindow,"shutdownDrainTimeWindow enabled" for signal in ['SIGINT', 'SIGHUP', 'SIGQUIT', 'SIGUSR1', 'SIGUSR2', 'SIGTERM', 'SIGABRT'] process.on signal, -> - if Settings.shutDownInProgress - logger.log signal: signal, "shutdown already in progress, ignoring signal" - return - else - Settings.shutDownInProgress = true - logger.warn signal: signal, "received interrupt, starting drain over #{shutdownDrainTimeWindow} mins" - DrainManager.startDrainTimeWindow(io, shutdownDrainTimeWindow) - shutdownCleanly(signal) - + drainAndShutdown(signal) + # global exception handler + if Settings.errors?.catchUncaughtErrors + process.removeAllListeners('uncaughtException') + process.on 'uncaughtException', (error) -> + logger.error err: error, 'uncaught exception' + if Settings.errors?.shutdownOnUncaughtError + drainAndShutdown('SIGABRT') if Settings.continualPubsubTraffic console.log "continualPubsubTraffic enabled" diff --git a/services/real-time/app/coffee/DrainManager.coffee b/services/real-time/app/coffee/DrainManager.coffee index ad142ccc47..2590a96726 100644 --- a/services/real-time/app/coffee/DrainManager.coffee +++ b/services/real-time/app/coffee/DrainManager.coffee @@ -36,4 +36,4 @@ module.exports = DrainManager = if haveDrainedNClients break if drainedCount < N - logger.log "All clients have been told to reconnectGracefully" \ No newline at end of file + logger.log "All clients have been told to reconnectGracefully" diff --git a/services/real-time/app/coffee/Router.coffee b/services/real-time/app/coffee/Router.coffee index 92a17729b7..ae4d404e48 100644 --- a/services/real-time/app/coffee/Router.coffee +++ b/services/real-time/app/coffee/Router.coffee @@ -42,6 +42,12 @@ module.exports = Router = app.post "/drain", httpAuth, HttpApiController.startDrain session.on 'connection', (error, client, session) -> + client?.on "error", (err) -> + logger.err "socket.io client error", { err } + if client.connected + client.emit("reconnectGracefully") + client.disconnect() + if settings.shutDownInProgress client.emit("connectionRejected", {message: "retry"}) client.disconnect() diff --git a/services/real-time/config/settings.defaults.coffee b/services/real-time/config/settings.defaults.coffee index 7fba6ea0d6..75b39ab2ee 100644 --- a/services/real-time/config/settings.defaults.coffee +++ b/services/real-time/config/settings.defaults.coffee @@ -62,6 +62,10 @@ settings = sentry: dsn: process.env.SENTRY_DSN + + errors: + catchUncaughtErrors: true + shutdownOnUncaughtError: true # console.log settings.redis module.exports = settings From ce366fdbeeb35955a107739262b7881a6f8dd974 Mon Sep 17 00:00:00 2001 From: Simon Detheridge Date: Thu, 17 Oct 2019 12:46:07 +0100 Subject: [PATCH 294/491] Bump Dockerfile to node 10 --- services/real-time/Dockerfile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/services/real-time/Dockerfile b/services/real-time/Dockerfile index e7243c5291..7656cf4526 100644 --- a/services/real-time/Dockerfile +++ b/services/real-time/Dockerfile @@ -1,4 +1,4 @@ -FROM node:6.15.1 as app +FROM node:10.16.3 as app WORKDIR /app @@ -12,7 +12,7 @@ COPY . /app RUN npm run compile:all -FROM node:6.15.1 +FROM node:10.16.3 COPY --from=app /app /app From 925a8651c1eec19ff42de36b068807c14481d058 Mon Sep 17 00:00:00 2001 From: Simon Detheridge Date: Tue, 22 Oct 2019 10:17:38 +0100 Subject: [PATCH 295/491] Revert "Track the `isRestrictedUser` flag on clients" --- .../real-time/app/coffee/WebApiManager.coffee | 8 +- .../app/coffee/WebsocketController.coffee | 35 +++-- .../app/coffee/WebsocketLoadBalancer.coffee | 41 +----- .../unit/coffee/WebApiManagerTests.coffee | 29 ++-- .../coffee/WebsocketControllerTests.coffee | 124 ++++++++---------- .../coffee/WebsocketLoadBalancerTests.coffee | 64 +-------- 6 files changed, 91 insertions(+), 210 deletions(-) diff --git a/services/real-time/app/coffee/WebApiManager.coffee b/services/real-time/app/coffee/WebApiManager.coffee index 3c0551a815..6836d5f213 100644 --- a/services/real-time/app/coffee/WebApiManager.coffee +++ b/services/real-time/app/coffee/WebApiManager.coffee @@ -4,7 +4,7 @@ logger = require "logger-sharelatex" { CodedError } = require "./Errors" module.exports = WebApiManager = - joinProject: (project_id, user, callback = (error, project, privilegeLevel, isRestrictedUser) ->) -> + joinProject: (project_id, user, callback = (error, project, privilegeLevel) ->) -> user_id = user._id logger.log {project_id, user_id}, "sending join project request to web" url = "#{settings.apis.web.url}/project/#{project_id}/join" @@ -24,11 +24,7 @@ module.exports = WebApiManager = }, (error, response, data) -> return callback(error) if error? if 200 <= response.statusCode < 300 - if !data? || !data?.project? - err = new Error('no data returned from joinProject request') - logger.error {err, project_id, user_id}, "error accessing web api" - return callback(err) - callback null, data.project, data.privilegeLevel, data.isRestrictedUser + callback null, data?.project, data?.privilegeLevel else if response.statusCode == 429 logger.log(project_id, user_id, "rate-limit hit when joining project") callback(new CodedError("rate-limit hit when joining project", "TooManyRequests")) diff --git a/services/real-time/app/coffee/WebsocketController.coffee b/services/real-time/app/coffee/WebsocketController.coffee index d0ca99cc5c..5b03f334d5 100644 --- a/services/real-time/app/coffee/WebsocketController.coffee +++ b/services/real-time/app/coffee/WebsocketController.coffee @@ -13,12 +13,12 @@ module.exports = WebsocketController = # it will force a full refresh of the page. Useful for non-backwards # compatible protocol changes. Use only in extreme need. PROTOCOL_VERSION: 2 - + joinProject: (client, user, project_id, callback = (error, project, privilegeLevel, protocolVersion) ->) -> user_id = user?._id logger.log {user_id, project_id, client_id: client.id}, "user joining project" metrics.inc "editor.join-project" - WebApiManager.joinProject project_id, user, (error, project, privilegeLevel, isRestrictedUser) -> + WebApiManager.joinProject project_id, user, (error, project, privilegeLevel) -> return callback(error) if error? if !privilegeLevel or privilegeLevel == "" @@ -26,6 +26,7 @@ module.exports = WebsocketController = logger.warn {err, project_id, user_id, client_id: client.id}, "user is not authorized to join project" return callback(err) + client.set("privilege_level", privilegeLevel) client.set("user_id", user_id) client.set("project_id", project_id) @@ -36,19 +37,18 @@ module.exports = WebsocketController = client.set("connected_time", new Date()) client.set("signup_date", user?.signUpDate) client.set("login_count", user?.loginCount) - client.set("is_restricted_user", !!(isRestrictedUser)) - + RoomManager.joinProject client, project_id, (err) -> logger.log {user_id, project_id, client_id: client.id}, "user joined project" callback null, project, privilegeLevel, WebsocketController.PROTOCOL_VERSION - + # No need to block for setting the user as connected in the cursor tracking ConnectedUsersManager.updateUserPosition project_id, client.id, user, null, () -> - + # We want to flush a project if there are no more (local) connected clients # but we need to wait for the triggering client to disconnect. How long we wait # is determined by FLUSH_IF_EMPTY_DELAY. - FLUSH_IF_EMPTY_DELAY: 500 #ms + FLUSH_IF_EMPTY_DELAY: 500 #ms leaveProject: (io, client, callback = (error) ->) -> metrics.inc "editor.leave-project" Utils.getClientAttributes client, ["project_id", "user_id"], (error, {project_id, user_id}) -> @@ -56,7 +56,7 @@ module.exports = WebsocketController = logger.log {project_id, user_id, client_id: client.id}, "client leaving project" WebsocketLoadBalancer.emitToRoom project_id, "clientTracking.clientDisconnected", client.id - + # bail out if the client had not managed to authenticate or join # the project. Prevents downstream errors in docupdater from # flushProjectToMongoAndDelete with null project_id. @@ -66,12 +66,12 @@ module.exports = WebsocketController = if not project_id? logger.log {user_id: user_id, client_id: client.id}, "client leaving, not in project" return callback() - + # We can do this in the background ConnectedUsersManager.markUserAsDisconnected project_id, client.id, (err) -> if err? logger.error {err, project_id, user_id, client_id: client.id}, "error marking client as disconnected" - + RoomManager.leaveProjectAndDocs(client) setTimeout () -> remainingClients = io.sockets.clients(project_id) @@ -82,14 +82,14 @@ module.exports = WebsocketController = logger.error {err, project_id, user_id, client_id: client.id}, "error flushing to doc updater after leaving project" callback() , WebsocketController.FLUSH_IF_EMPTY_DELAY - + joinDoc: (client, doc_id, fromVersion = -1, options, callback = (error, doclines, version, ops, ranges) ->) -> metrics.inc "editor.join-doc" Utils.getClientAttributes client, ["project_id", "user_id"], (error, {project_id, user_id}) -> return callback(error) if error? return callback(new Error("no project_id found on client")) if !project_id? logger.log {user_id, project_id, doc_id, fromVersion, client_id: client.id}, "client joining doc" - + AuthorizationManager.assertClientCanViewProject client, (error) -> return callback(error) if error? # ensure the per-doc applied-ops channel is subscribed before sending the @@ -124,7 +124,7 @@ module.exports = WebsocketController = AuthorizationManager.addAccessToDoc client, doc_id logger.log {user_id, project_id, doc_id, fromVersion, client_id: client.id}, "client joined doc" callback null, escapedLines, version, ops, ranges - + leaveDoc: (client, doc_id, callback = (error) ->) -> metrics.inc "editor.leave-doc" Utils.getClientAttributes client, ["project_id", "user_id"], (error, {project_id, user_id}) -> @@ -178,11 +178,8 @@ module.exports = WebsocketController = CLIENT_REFRESH_DELAY: 1000 getConnectedUsers: (client, callback = (error, users) ->) -> metrics.inc "editor.get-connected-users" - Utils.getClientAttributes client, ["project_id", "user_id", "is_restricted_user"], (error, clientAttributes) -> + Utils.getClientAttributes client, ["project_id", "user_id"], (error, {project_id, user_id}) -> return callback(error) if error? - {project_id, user_id, is_restricted_user} = clientAttributes - if is_restricted_user - return callback(null, []) return callback(new Error("no project_id found on client")) if !project_id? logger.log {user_id, project_id, client_id: client.id}, "getting connected users" AuthorizationManager.assertClientCanViewProject client, (error) -> @@ -230,9 +227,9 @@ module.exports = WebsocketController = return callback(error) else return callback(null) - + _isCommentUpdate: (update) -> for op in update.op if !op.c? return false - return true + return true \ No newline at end of file diff --git a/services/real-time/app/coffee/WebsocketLoadBalancer.coffee b/services/real-time/app/coffee/WebsocketLoadBalancer.coffee index 12b7ef812b..45a7645005 100644 --- a/services/real-time/app/coffee/WebsocketLoadBalancer.coffee +++ b/services/real-time/app/coffee/WebsocketLoadBalancer.coffee @@ -7,19 +7,6 @@ HealthCheckManager = require "./HealthCheckManager" RoomManager = require "./RoomManager" ChannelManager = require "./ChannelManager" ConnectedUsersManager = require "./ConnectedUsersManager" -Utils = require './Utils' -Async = require 'async' - -RESTRICTED_USER_MESSAGE_TYPE_PASS_LIST = [ - 'connectionAccepted', - 'otUpdateApplied', - 'otUpdateError', - 'joinDoc', - 'reciveNewDoc', - 'reciveNewFile', - 'reciveNewFolder', - 'removeEntity' -] module.exports = WebsocketLoadBalancer = rclientPubList: RedisClientManager.createClientList(Settings.redis.pubsub) @@ -71,7 +58,7 @@ module.exports = WebsocketLoadBalancer = else if message.message is 'clientTracking.refresh' && message.room_id? clientList = io.sockets.clients(message.room_id) logger.log {channel:channel, message: message.message, room_id: message.room_id, message_id: message._id, socketIoClients: (client.id for client in clientList)}, "refreshing client list" - for client in clientList + for client in clientList ConnectedUsersManager.refreshClient(message.room_id, client.id) else if message.room_id? if message._id? && Settings.checkEventOrder @@ -82,28 +69,12 @@ module.exports = WebsocketLoadBalancer = clientList = io.sockets.clients(message.room_id) # avoid unnecessary work if no clients are connected return if clientList.length is 0 - logger.log { - channel: channel, - message: message.message, - room_id: message.room_id, - message_id: message._id, - socketIoClients: (client.id for client in clientList) - }, "distributing event to clients" + logger.log {channel:channel, message: message.message, room_id: message.room_id, message_id: message._id, socketIoClients: (client.id for client in clientList)}, "distributing event to clients" seen = {} - # Send the messages to clients async, don't wait for them all to finish - Async.eachLimit clientList - , 2 - , (client, cb) -> - Utils.getClientAttributes client, ['is_restricted_user'], (err, {is_restricted_user}) -> - return cb(err) if err? - if !seen[client.id] - seen[client.id] = true - if !(is_restricted_user && message.message not in RESTRICTED_USER_MESSAGE_TYPE_PASS_LIST) - client.emit(message.message, message.payload...) - cb() - , (err) -> - if err? - logger.err {err, message}, "Error sending message to clients" + for client in clientList when not seen[client.id] + seen[client.id] = true + client.emit(message.message, message.payload...) else if message.health_check? logger.debug {message}, "got health check message in editor events channel" HealthCheckManager.check channel, message.key + diff --git a/services/real-time/test/unit/coffee/WebApiManagerTests.coffee b/services/real-time/test/unit/coffee/WebApiManagerTests.coffee index a87522c0a9..278585518d 100644 --- a/services/real-time/test/unit/coffee/WebApiManagerTests.coffee +++ b/services/real-time/test/unit/coffee/WebApiManagerTests.coffee @@ -20,18 +20,17 @@ describe 'WebApiManager', -> user: "username" pass: "password" "logger-sharelatex": @logger = { log: sinon.stub(), error: sinon.stub() } - + describe "joinProject", -> describe "successfully", -> beforeEach -> @response = { project: { name: "Test project" } - privilegeLevel: "owner", - isRestrictedUser: true + privilegeLevel: "owner" } @request.post = sinon.stub().callsArgWith(1, null, {statusCode: 200}, @response) @WebApiManager.joinProject @project_id, @user, @callback - + it "should send a request to web to join the project", -> @request.post .calledWith({ @@ -47,32 +46,22 @@ describe 'WebApiManager', -> headers: {} }) .should.equal true - - it "should return the project, privilegeLevel, and restricted flag", -> + + it "should return the project and privilegeLevel", -> @callback - .calledWith(null, @response.project, @response.privilegeLevel, @response.isRestrictedUser) + .calledWith(null, @response.project, @response.privilegeLevel) .should.equal true - + describe "with an error from web", -> beforeEach -> @request.post = sinon.stub().callsArgWith(1, null, {statusCode: 500}, null) @WebApiManager.joinProject @project_id, @user_id, @callback - + it "should call the callback with an error", -> @callback .calledWith(new Error("non-success code from web: 500")) .should.equal true - - describe "with no data from web", -> - beforeEach -> - @request.post = sinon.stub().callsArgWith(1, null, {statusCode: 200}, null) - @WebApiManager.joinProject @project_id, @user_id, @callback - - it "should call the callback with an error", -> - @callback - .calledWith(new Error("no data returned from joinProject request")) - .should.equal true - + describe "when the project is over its rate limit", -> beforeEach -> @request.post = sinon.stub().callsArgWith(1, null, {statusCode: 429}, null) diff --git a/services/real-time/test/unit/coffee/WebsocketControllerTests.coffee b/services/real-time/test/unit/coffee/WebsocketControllerTests.coffee index 116485384d..edb0055bf5 100644 --- a/services/real-time/test/unit/coffee/WebsocketControllerTests.coffee +++ b/services/real-time/test/unit/coffee/WebsocketControllerTests.coffee @@ -37,10 +37,10 @@ describe 'WebsocketController', -> inc: sinon.stub() set: sinon.stub() "./RoomManager": @RoomManager = {} - + afterEach -> tk.reset() - + describe "joinProject", -> describe "when authorised", -> beforeEach -> @@ -53,65 +53,61 @@ describe 'WebsocketController', -> } @privilegeLevel = "owner" @ConnectedUsersManager.updateUserPosition = sinon.stub().callsArg(4) - @isRestrictedUser = true - @WebApiManager.joinProject = sinon.stub().callsArgWith(2, null, @project, @privilegeLevel, @isRestrictedUser) + @WebApiManager.joinProject = sinon.stub().callsArgWith(2, null, @project, @privilegeLevel) @RoomManager.joinProject = sinon.stub().callsArg(2) @WebsocketController.joinProject @client, @user, @project_id, @callback - + it "should load the project from web", -> @WebApiManager.joinProject .calledWith(@project_id, @user) .should.equal true - + it "should join the project room", -> @RoomManager.joinProject.calledWith(@client, @project_id).should.equal true - + it "should set the privilege level on the client", -> @client.set.calledWith("privilege_level", @privilegeLevel).should.equal true - + it "should set the user's id on the client", -> @client.set.calledWith("user_id", @user._id).should.equal true - + it "should set the user's email on the client", -> @client.set.calledWith("email", @user.email).should.equal true - + it "should set the user's first_name on the client", -> @client.set.calledWith("first_name", @user.first_name).should.equal true - + it "should set the user's last_name on the client", -> @client.set.calledWith("last_name", @user.last_name).should.equal true - + it "should set the user's sign up date on the client", -> @client.set.calledWith("signup_date", @user.signUpDate).should.equal true - + it "should set the user's login_count on the client", -> @client.set.calledWith("login_count", @user.loginCount).should.equal true - + it "should set the connected time on the client", -> @client.set.calledWith("connected_time", new Date()).should.equal true - + it "should set the project_id on the client", -> @client.set.calledWith("project_id", @project_id).should.equal true - + it "should set the project owner id on the client", -> @client.set.calledWith("owner_id", @owner_id).should.equal true - - it "should set the is_restricted_user flag on the client", -> - @client.set.calledWith("is_restricted_user", @isRestrictedUser).should.equal true - + it "should call the callback with the project, privilegeLevel and protocolVersion", -> @callback .calledWith(null, @project, @privilegeLevel, @WebsocketController.PROTOCOL_VERSION) .should.equal true - + it "should mark the user as connected in ConnectedUsersManager", -> @ConnectedUsersManager.updateUserPosition .calledWith(@project_id, @client.id, @user, null) .should.equal true - + it "should increment the join-project metric", -> @metrics.inc.calledWith("editor.join-project").should.equal true - + describe "when not authorized", -> beforeEach -> @WebApiManager.joinProject = sinon.stub().callsArgWith(2, null, null, null) @@ -142,30 +138,30 @@ describe 'WebsocketController', -> @client.params.user_id = @user_id @WebsocketController.FLUSH_IF_EMPTY_DELAY = 0 tk.reset() # Allow setTimeout to work. - + describe "when the project is empty", -> beforeEach (done) -> @clientsInRoom = [] @WebsocketController.leaveProject @io, @client, done - + it "should end clientTracking.clientDisconnected to the project room", -> @WebsocketLoadBalancer.emitToRoom .calledWith(@project_id, "clientTracking.clientDisconnected", @client.id) .should.equal true - + it "should mark the user as disconnected", -> @ConnectedUsersManager.markUserAsDisconnected .calledWith(@project_id, @client.id) .should.equal true - + it "should flush the project in the document updater", -> @DocumentUpdaterManager.flushProjectToMongoAndDelete .calledWith(@project_id) .should.equal true - + it "should increment the leave-project metric", -> @metrics.inc.calledWith("editor.leave-project").should.equal true - + it "should track the disconnection in RoomManager", -> @RoomManager.leaveProjectAndDocs .calledWith(@client) @@ -175,7 +171,7 @@ describe 'WebsocketController', -> beforeEach -> @clientsInRoom = ["mock-remaining-client"] @WebsocketController.leaveProject @io, @client - + it "should not flush the project in the document updater", -> @DocumentUpdaterManager.flushProjectToMongoAndDelete .called.should.equal false @@ -236,7 +232,7 @@ describe 'WebsocketController', -> @ops = ["mock", "ops"] @ranges = { "mock": "ranges" } @options = {} - + @client.params.project_id = @project_id @AuthorizationManager.addAccessToDoc = sinon.stub() @AuthorizationManager.assertClientCanViewProject = sinon.stub().callsArgWith(1, null) @@ -279,7 +275,7 @@ describe 'WebsocketController', -> beforeEach -> @fromVersion = 40 @WebsocketController.joinDoc @client, @doc_id, @fromVersion, @options, @callback - + it "should get the document from the DocumentUpdaterManager with fromVersion", -> @DocumentUpdaterManager.getDocument .calledWith(@project_id, @doc_id, @fromVersion) @@ -289,7 +285,7 @@ describe 'WebsocketController', -> beforeEach -> @doc_lines.push ["räksmörgås"] @WebsocketController.joinDoc @client, @doc_id, -1, @options, @callback - + it "should call the callback with the escaped lines", -> escaped_lines = @callback.args[0][1] escaped_word = escaped_lines.pop() @@ -331,44 +327,44 @@ describe 'WebsocketController', -> beforeEach -> @AuthorizationManager.assertClientCanViewProject = sinon.stub().callsArgWith(1, @err = new Error("not authorized")) @WebsocketController.joinDoc @client, @doc_id, -1, @options, @callback - + it "should call the callback with an error", -> @callback.calledWith(@err).should.equal true - + it "should not call the DocumentUpdaterManager", -> @DocumentUpdaterManager.getDocument.called.should.equal false - + describe "leaveDoc", -> beforeEach -> - @doc_id = "doc-id-123" + @doc_id = "doc-id-123" @client.params.project_id = @project_id @RoomManager.leaveDoc = sinon.stub() @WebsocketController.leaveDoc @client, @doc_id, @callback - + it "should remove the client from the doc_id room", -> @RoomManager.leaveDoc .calledWith(@client, @doc_id).should.equal true - + it "should call the callback", -> @callback.called.should.equal true - + it "should increment the leave-doc metric", -> @metrics.inc.calledWith("editor.leave-doc").should.equal true - + describe "getConnectedUsers", -> beforeEach -> @client.params.project_id = @project_id @users = ["mock", "users"] @WebsocketLoadBalancer.emitToRoom = sinon.stub() @ConnectedUsersManager.getConnectedUsers = sinon.stub().callsArgWith(1, null, @users) - + describe "when authorized", -> beforeEach (done) -> @AuthorizationManager.assertClientCanViewProject = sinon.stub().callsArgWith(1, null) @WebsocketController.getConnectedUsers @client, (args...) => @callback(args...) done() - + it "should check that the client is authorized to view the project", -> @AuthorizationManager.assertClientCanViewProject .calledWith(@client) @@ -383,40 +379,26 @@ describe 'WebsocketController', -> @ConnectedUsersManager.getConnectedUsers .calledWith(@project_id) .should.equal true - + it "should return the users", -> @callback.calledWith(null, @users).should.equal true - + it "should increment the get-connected-users metric", -> @metrics.inc.calledWith("editor.get-connected-users").should.equal true - + describe "when not authorized", -> beforeEach -> @AuthorizationManager.assertClientCanViewProject = sinon.stub().callsArgWith(1, @err = new Error("not authorized")) @WebsocketController.getConnectedUsers @client, @callback - + it "should not get the connected users for the project", -> @ConnectedUsersManager.getConnectedUsers .called .should.equal false - + it "should return an error", -> @callback.calledWith(@err).should.equal true - - describe "when restricted user", -> - beforeEach -> - @client.params.is_restricted_user = true - @AuthorizationManager.assertClientCanViewProject = sinon.stub().callsArgWith(1, null) - @WebsocketController.getConnectedUsers @client, @callback - - it "should return an empty array of users", -> - @callback.calledWith(null, []).should.equal true - - it "should not get the connected users for the project", -> - @ConnectedUsersManager.getConnectedUsers - .called - .should.equal false - + describe "updateClientPosition", -> beforeEach -> @WebsocketLoadBalancer.emitToRoom = sinon.stub() @@ -440,7 +422,7 @@ describe 'WebsocketController', -> @client.get = (param, callback) => callback null, @clientParams[param] @WebsocketController.updateClientPosition @client, @update - @populatedCursorData = + @populatedCursorData = doc_id: @doc_id, id: @client.id name: "#{@first_name} #{@last_name}" @@ -480,7 +462,7 @@ describe 'WebsocketController', -> @client.get = (param, callback) => callback null, @clientParams[param] @WebsocketController.updateClientPosition @client, @update - @populatedCursorData = + @populatedCursorData = doc_id: @doc_id, id: @client.id name: "#{@first_name}" @@ -520,7 +502,7 @@ describe 'WebsocketController', -> @client.get = (param, callback) => callback null, @clientParams[param] @WebsocketController.updateClientPosition @client, @update - @populatedCursorData = + @populatedCursorData = doc_id: @doc_id, id: @client.id name: "#{@last_name}" @@ -621,7 +603,7 @@ describe 'WebsocketController', -> it "should call the callback", -> @callback.called.should.equal true - + it "should increment the doc updates", -> @metrics.inc.calledWith("editor.doc-update").should.equal true @@ -639,7 +621,7 @@ describe 'WebsocketController', -> it "should call the callback with the error", -> @callback.calledWith(@error).should.equal true - + describe "when not authorized", -> beforeEach -> @client.disconnect = sinon.stub() @@ -663,7 +645,7 @@ describe 'WebsocketController', -> @comment_update = { op: [{c: "bar", p: 132}] } @AuthorizationManager.assertClientCanEditProjectAndDoc = sinon.stub() @AuthorizationManager.assertClientCanViewProjectAndDoc = sinon.stub() - + describe "with a read-write client", -> it "should return successfully", (done) -> @AuthorizationManager.assertClientCanEditProjectAndDoc.yields(null) @@ -678,7 +660,7 @@ describe 'WebsocketController', -> @WebsocketController._assertClientCanApplyUpdate @client, @doc_id, @edit_update, (error) -> expect(error.message).to.equal "not authorized" done() - + describe "with a read-only client and a comment op", -> it "should return successfully", (done) -> @AuthorizationManager.assertClientCanEditProjectAndDoc.yields(new Error("not authorized")) @@ -686,7 +668,7 @@ describe 'WebsocketController', -> @WebsocketController._assertClientCanApplyUpdate @client, @doc_id, @comment_update, (error) -> expect(error).to.be.null done() - + describe "with a totally unauthorized client", -> it "should return an error", (done) -> @AuthorizationManager.assertClientCanEditProjectAndDoc.yields(new Error("not authorized")) diff --git a/services/real-time/test/unit/coffee/WebsocketLoadBalancerTests.coffee b/services/real-time/test/unit/coffee/WebsocketLoadBalancerTests.coffee index e6fe1df8c9..66e950cd0e 100644 --- a/services/real-time/test/unit/coffee/WebsocketLoadBalancerTests.coffee +++ b/services/real-time/test/unit/coffee/WebsocketLoadBalancerTests.coffee @@ -8,7 +8,7 @@ describe "WebsocketLoadBalancer", -> @rclient = {} @RoomEvents = {on: sinon.stub()} @WebsocketLoadBalancer = SandboxedModule.require modulePath, requires: - "./RedisClientManager": + "./RedisClientManager": createClientList: () => [] "logger-sharelatex": @logger = { log: sinon.stub(), error: sinon.stub() } "./SafeJsonParse": @SafeJsonParse = @@ -18,12 +18,6 @@ describe "WebsocketLoadBalancer", -> "./RoomManager" : @RoomManager = {eventSource: sinon.stub().returns @RoomEvents} "./ChannelManager": @ChannelManager = {publish: sinon.stub()} "./ConnectedUsersManager": @ConnectedUsersManager = {refreshClient: sinon.stub()} - "./Utils": @Utils = { - getClientAttributes: sinon.spy( - (client, _attrs, callback) -> - callback(null, {is_restricted_user: !!client.__isRestricted}) - ) - } @io = {} @WebsocketLoadBalancer.rclientPubList = [{publish: sinon.stub()}] @WebsocketLoadBalancer.rclientSubList = [{ @@ -32,7 +26,7 @@ describe "WebsocketLoadBalancer", -> }] @room_id = "room-id" - @message = "otUpdateApplied" + @message = "message-to-editor" @payload = ["argument one", 42] describe "emitToRoom", -> @@ -57,7 +51,7 @@ describe "WebsocketLoadBalancer", -> @WebsocketLoadBalancer.emitToRoom .calledWith("all", @message, @payload...) .should.equal true - + describe "listenForEditorEvents", -> beforeEach -> @WebsocketLoadBalancer._processEditorEvent = sinon.stub() @@ -76,10 +70,9 @@ describe "WebsocketLoadBalancer", -> describe "_processEditorEvent", -> describe "with bad JSON", -> beforeEach -> - @isRestrictedUser = false @SafeJsonParse.parse = sinon.stub().callsArgWith 1, new Error("oops") @WebsocketLoadBalancer._processEditorEvent(@io, "editor-events", "blah") - + it "should log an error", -> @logger.error.called.should.equal true @@ -105,54 +98,6 @@ describe "WebsocketLoadBalancer", -> @emit2.calledWith(@message, @payload...).should.equal true @emit3.called.should.equal false # duplicate client should be ignored - describe "with a designated room, and restricted clients, not restricted message", -> - beforeEach -> - @io.sockets = - clients: sinon.stub().returns([ - {id: 'client-id-1', emit: @emit1 = sinon.stub()} - {id: 'client-id-2', emit: @emit2 = sinon.stub()} - {id: 'client-id-1', emit: @emit3 = sinon.stub()} # duplicate client - {id: 'client-id-4', emit: @emit4 = sinon.stub(), __isRestricted: true} - ]) - data = JSON.stringify - room_id: @room_id - message: @message - payload: @payload - @WebsocketLoadBalancer._processEditorEvent(@io, "editor-events", data) - - it "should send the message to all (unique) clients in the room", -> - @io.sockets.clients - .calledWith(@room_id) - .should.equal true - @emit1.calledWith(@message, @payload...).should.equal true - @emit2.calledWith(@message, @payload...).should.equal true - @emit3.called.should.equal false # duplicate client should be ignored - @emit4.called.should.equal true # restricted client, but should be called - - describe "with a designated room, and restricted clients, restricted message", -> - beforeEach -> - @io.sockets = - clients: sinon.stub().returns([ - {id: 'client-id-1', emit: @emit1 = sinon.stub()} - {id: 'client-id-2', emit: @emit2 = sinon.stub()} - {id: 'client-id-1', emit: @emit3 = sinon.stub()} # duplicate client - {id: 'client-id-4', emit: @emit4 = sinon.stub(), __isRestricted: true} - ]) - data = JSON.stringify - room_id: @room_id - message: @restrictedMessage = 'new-comment' - payload: @payload - @WebsocketLoadBalancer._processEditorEvent(@io, "editor-events", data) - - it "should send the message to all (unique) clients in the room, who are not restricted", -> - @io.sockets.clients - .calledWith(@room_id) - .should.equal true - @emit1.calledWith(@restrictedMessage, @payload...).should.equal true - @emit2.calledWith(@restrictedMessage, @payload...).should.equal true - @emit3.called.should.equal false # duplicate client should be ignored - @emit4.called.should.equal false # restricted client, should not be called - describe "when emitting to all", -> beforeEach -> @io.sockets = @@ -165,3 +110,4 @@ describe "WebsocketLoadBalancer", -> it "should send the message to all clients", -> @emit.calledWith(@message, @payload...).should.equal true + From 3dc7c357a5299044b8e7d1534c4d6219af7b63e6 Mon Sep 17 00:00:00 2001 From: Nate Stemen Date: Fri, 25 Oct 2019 13:22:58 -0400 Subject: [PATCH 296/491] add public link to contributing docs --- services/real-time/.github/PULL_REQUEST_TEMPLATE.md | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/services/real-time/.github/PULL_REQUEST_TEMPLATE.md b/services/real-time/.github/PULL_REQUEST_TEMPLATE.md index ed25ee83c1..12bb2eeb3f 100644 --- a/services/real-time/.github/PULL_REQUEST_TEMPLATE.md +++ b/services/real-time/.github/PULL_REQUEST_TEMPLATE.md @@ -1,4 +1,7 @@ - + + + + ### Description From 9a838bd071326c5a7a90442051ba9d0f7006ea94 Mon Sep 17 00:00:00 2001 From: Nate Stemen Date: Fri, 25 Oct 2019 13:23:13 -0400 Subject: [PATCH 297/491] bump build script to 1.1.24 --- services/real-time/Makefile | 4 ++-- services/real-time/buildscript.txt | 4 +++- services/real-time/docker-compose.ci.yml | 2 +- services/real-time/docker-compose.yml | 2 +- 4 files changed, 7 insertions(+), 5 deletions(-) diff --git a/services/real-time/Makefile b/services/real-time/Makefile index 573659dd90..487735cf19 100644 --- a/services/real-time/Makefile +++ b/services/real-time/Makefile @@ -1,7 +1,7 @@ # This file was auto-generated, do not edit it directly. # Instead run bin/update_build_scripts from # https://github.com/sharelatex/sharelatex-dev-environment -# Version: 1.1.21 +# Version: 1.1.24 BUILD_NUMBER ?= local BRANCH_NAME ?= $(shell git rev-parse --abbrev-ref HEAD) @@ -35,7 +35,7 @@ test_clean: $(DOCKER_COMPOSE) down -v -t 0 test_acceptance_pre_run: - @[ ! -f test/acceptance/scripts/pre-run ] && echo "real-time has no pre acceptance tests task" || $(DOCKER_COMPOSE) run --rm test_acceptance test/acceptance/scripts/pre-run + @[ ! -f test/acceptance/js/scripts/pre-run ] && echo "real-time has no pre acceptance tests task" || $(DOCKER_COMPOSE) run --rm test_acceptance test/acceptance/js/scripts/pre-run build: docker build --pull --tag ci/$(PROJECT_NAME):$(BRANCH_NAME)-$(BUILD_NUMBER) \ --tag gcr.io/overleaf-ops/$(PROJECT_NAME):$(BRANCH_NAME)-$(BUILD_NUMBER) \ diff --git a/services/real-time/buildscript.txt b/services/real-time/buildscript.txt index 1f61f10934..1818ad74a9 100644 --- a/services/real-time/buildscript.txt +++ b/services/real-time/buildscript.txt @@ -5,4 +5,6 @@ real-time --dependencies=mongo,redis --docker-repos=gcr.io/overleaf-ops --build-target=docker ---script-version=1.1.21 +--script-version=1.1.24 +--env-pass-through= +--public-repo=True diff --git a/services/real-time/docker-compose.ci.yml b/services/real-time/docker-compose.ci.yml index d2bcca9ec6..c78d90e8ed 100644 --- a/services/real-time/docker-compose.ci.yml +++ b/services/real-time/docker-compose.ci.yml @@ -1,7 +1,7 @@ # This file was auto-generated, do not edit it directly. # Instead run bin/update_build_scripts from # https://github.com/sharelatex/sharelatex-dev-environment -# Version: 1.1.21 +# Version: 1.1.24 version: "2" diff --git a/services/real-time/docker-compose.yml b/services/real-time/docker-compose.yml index 47b12619da..172de89b22 100644 --- a/services/real-time/docker-compose.yml +++ b/services/real-time/docker-compose.yml @@ -1,7 +1,7 @@ # This file was auto-generated, do not edit it directly. # Instead run bin/update_build_scripts from # https://github.com/sharelatex/sharelatex-dev-environment -# Version: 1.1.21 +# Version: 1.1.24 version: "2" From 403caa65e8f8a8f10bb7bfa380fb74bb3a5a1cf8 Mon Sep 17 00:00:00 2001 From: Shane Kilkelly Date: Wed, 30 Oct 2019 13:52:36 +0000 Subject: [PATCH 298/491] Revert "Revert "Track the `isRestrictedUser` flag on clients"" This reverts commit 651e392a7c644403f199e1b03e7494b61ce71d0c. --- .../real-time/app/coffee/WebApiManager.coffee | 8 +- .../app/coffee/WebsocketController.coffee | 35 ++--- .../app/coffee/WebsocketLoadBalancer.coffee | 41 +++++- .../unit/coffee/WebApiManagerTests.coffee | 29 ++-- .../coffee/WebsocketControllerTests.coffee | 124 ++++++++++-------- .../coffee/WebsocketLoadBalancerTests.coffee | 64 ++++++++- 6 files changed, 210 insertions(+), 91 deletions(-) diff --git a/services/real-time/app/coffee/WebApiManager.coffee b/services/real-time/app/coffee/WebApiManager.coffee index 6836d5f213..3c0551a815 100644 --- a/services/real-time/app/coffee/WebApiManager.coffee +++ b/services/real-time/app/coffee/WebApiManager.coffee @@ -4,7 +4,7 @@ logger = require "logger-sharelatex" { CodedError } = require "./Errors" module.exports = WebApiManager = - joinProject: (project_id, user, callback = (error, project, privilegeLevel) ->) -> + joinProject: (project_id, user, callback = (error, project, privilegeLevel, isRestrictedUser) ->) -> user_id = user._id logger.log {project_id, user_id}, "sending join project request to web" url = "#{settings.apis.web.url}/project/#{project_id}/join" @@ -24,7 +24,11 @@ module.exports = WebApiManager = }, (error, response, data) -> return callback(error) if error? if 200 <= response.statusCode < 300 - callback null, data?.project, data?.privilegeLevel + if !data? || !data?.project? + err = new Error('no data returned from joinProject request') + logger.error {err, project_id, user_id}, "error accessing web api" + return callback(err) + callback null, data.project, data.privilegeLevel, data.isRestrictedUser else if response.statusCode == 429 logger.log(project_id, user_id, "rate-limit hit when joining project") callback(new CodedError("rate-limit hit when joining project", "TooManyRequests")) diff --git a/services/real-time/app/coffee/WebsocketController.coffee b/services/real-time/app/coffee/WebsocketController.coffee index 5b03f334d5..d0ca99cc5c 100644 --- a/services/real-time/app/coffee/WebsocketController.coffee +++ b/services/real-time/app/coffee/WebsocketController.coffee @@ -13,12 +13,12 @@ module.exports = WebsocketController = # it will force a full refresh of the page. Useful for non-backwards # compatible protocol changes. Use only in extreme need. PROTOCOL_VERSION: 2 - + joinProject: (client, user, project_id, callback = (error, project, privilegeLevel, protocolVersion) ->) -> user_id = user?._id logger.log {user_id, project_id, client_id: client.id}, "user joining project" metrics.inc "editor.join-project" - WebApiManager.joinProject project_id, user, (error, project, privilegeLevel) -> + WebApiManager.joinProject project_id, user, (error, project, privilegeLevel, isRestrictedUser) -> return callback(error) if error? if !privilegeLevel or privilegeLevel == "" @@ -26,7 +26,6 @@ module.exports = WebsocketController = logger.warn {err, project_id, user_id, client_id: client.id}, "user is not authorized to join project" return callback(err) - client.set("privilege_level", privilegeLevel) client.set("user_id", user_id) client.set("project_id", project_id) @@ -37,18 +36,19 @@ module.exports = WebsocketController = client.set("connected_time", new Date()) client.set("signup_date", user?.signUpDate) client.set("login_count", user?.loginCount) - + client.set("is_restricted_user", !!(isRestrictedUser)) + RoomManager.joinProject client, project_id, (err) -> logger.log {user_id, project_id, client_id: client.id}, "user joined project" callback null, project, privilegeLevel, WebsocketController.PROTOCOL_VERSION - + # No need to block for setting the user as connected in the cursor tracking ConnectedUsersManager.updateUserPosition project_id, client.id, user, null, () -> - + # We want to flush a project if there are no more (local) connected clients # but we need to wait for the triggering client to disconnect. How long we wait # is determined by FLUSH_IF_EMPTY_DELAY. - FLUSH_IF_EMPTY_DELAY: 500 #ms + FLUSH_IF_EMPTY_DELAY: 500 #ms leaveProject: (io, client, callback = (error) ->) -> metrics.inc "editor.leave-project" Utils.getClientAttributes client, ["project_id", "user_id"], (error, {project_id, user_id}) -> @@ -56,7 +56,7 @@ module.exports = WebsocketController = logger.log {project_id, user_id, client_id: client.id}, "client leaving project" WebsocketLoadBalancer.emitToRoom project_id, "clientTracking.clientDisconnected", client.id - + # bail out if the client had not managed to authenticate or join # the project. Prevents downstream errors in docupdater from # flushProjectToMongoAndDelete with null project_id. @@ -66,12 +66,12 @@ module.exports = WebsocketController = if not project_id? logger.log {user_id: user_id, client_id: client.id}, "client leaving, not in project" return callback() - + # We can do this in the background ConnectedUsersManager.markUserAsDisconnected project_id, client.id, (err) -> if err? logger.error {err, project_id, user_id, client_id: client.id}, "error marking client as disconnected" - + RoomManager.leaveProjectAndDocs(client) setTimeout () -> remainingClients = io.sockets.clients(project_id) @@ -82,14 +82,14 @@ module.exports = WebsocketController = logger.error {err, project_id, user_id, client_id: client.id}, "error flushing to doc updater after leaving project" callback() , WebsocketController.FLUSH_IF_EMPTY_DELAY - + joinDoc: (client, doc_id, fromVersion = -1, options, callback = (error, doclines, version, ops, ranges) ->) -> metrics.inc "editor.join-doc" Utils.getClientAttributes client, ["project_id", "user_id"], (error, {project_id, user_id}) -> return callback(error) if error? return callback(new Error("no project_id found on client")) if !project_id? logger.log {user_id, project_id, doc_id, fromVersion, client_id: client.id}, "client joining doc" - + AuthorizationManager.assertClientCanViewProject client, (error) -> return callback(error) if error? # ensure the per-doc applied-ops channel is subscribed before sending the @@ -124,7 +124,7 @@ module.exports = WebsocketController = AuthorizationManager.addAccessToDoc client, doc_id logger.log {user_id, project_id, doc_id, fromVersion, client_id: client.id}, "client joined doc" callback null, escapedLines, version, ops, ranges - + leaveDoc: (client, doc_id, callback = (error) ->) -> metrics.inc "editor.leave-doc" Utils.getClientAttributes client, ["project_id", "user_id"], (error, {project_id, user_id}) -> @@ -178,8 +178,11 @@ module.exports = WebsocketController = CLIENT_REFRESH_DELAY: 1000 getConnectedUsers: (client, callback = (error, users) ->) -> metrics.inc "editor.get-connected-users" - Utils.getClientAttributes client, ["project_id", "user_id"], (error, {project_id, user_id}) -> + Utils.getClientAttributes client, ["project_id", "user_id", "is_restricted_user"], (error, clientAttributes) -> return callback(error) if error? + {project_id, user_id, is_restricted_user} = clientAttributes + if is_restricted_user + return callback(null, []) return callback(new Error("no project_id found on client")) if !project_id? logger.log {user_id, project_id, client_id: client.id}, "getting connected users" AuthorizationManager.assertClientCanViewProject client, (error) -> @@ -227,9 +230,9 @@ module.exports = WebsocketController = return callback(error) else return callback(null) - + _isCommentUpdate: (update) -> for op in update.op if !op.c? return false - return true \ No newline at end of file + return true diff --git a/services/real-time/app/coffee/WebsocketLoadBalancer.coffee b/services/real-time/app/coffee/WebsocketLoadBalancer.coffee index 45a7645005..12b7ef812b 100644 --- a/services/real-time/app/coffee/WebsocketLoadBalancer.coffee +++ b/services/real-time/app/coffee/WebsocketLoadBalancer.coffee @@ -7,6 +7,19 @@ HealthCheckManager = require "./HealthCheckManager" RoomManager = require "./RoomManager" ChannelManager = require "./ChannelManager" ConnectedUsersManager = require "./ConnectedUsersManager" +Utils = require './Utils' +Async = require 'async' + +RESTRICTED_USER_MESSAGE_TYPE_PASS_LIST = [ + 'connectionAccepted', + 'otUpdateApplied', + 'otUpdateError', + 'joinDoc', + 'reciveNewDoc', + 'reciveNewFile', + 'reciveNewFolder', + 'removeEntity' +] module.exports = WebsocketLoadBalancer = rclientPubList: RedisClientManager.createClientList(Settings.redis.pubsub) @@ -58,7 +71,7 @@ module.exports = WebsocketLoadBalancer = else if message.message is 'clientTracking.refresh' && message.room_id? clientList = io.sockets.clients(message.room_id) logger.log {channel:channel, message: message.message, room_id: message.room_id, message_id: message._id, socketIoClients: (client.id for client in clientList)}, "refreshing client list" - for client in clientList + for client in clientList ConnectedUsersManager.refreshClient(message.room_id, client.id) else if message.room_id? if message._id? && Settings.checkEventOrder @@ -69,12 +82,28 @@ module.exports = WebsocketLoadBalancer = clientList = io.sockets.clients(message.room_id) # avoid unnecessary work if no clients are connected return if clientList.length is 0 - logger.log {channel:channel, message: message.message, room_id: message.room_id, message_id: message._id, socketIoClients: (client.id for client in clientList)}, "distributing event to clients" + logger.log { + channel: channel, + message: message.message, + room_id: message.room_id, + message_id: message._id, + socketIoClients: (client.id for client in clientList) + }, "distributing event to clients" seen = {} - for client in clientList when not seen[client.id] - seen[client.id] = true - client.emit(message.message, message.payload...) + # Send the messages to clients async, don't wait for them all to finish + Async.eachLimit clientList + , 2 + , (client, cb) -> + Utils.getClientAttributes client, ['is_restricted_user'], (err, {is_restricted_user}) -> + return cb(err) if err? + if !seen[client.id] + seen[client.id] = true + if !(is_restricted_user && message.message not in RESTRICTED_USER_MESSAGE_TYPE_PASS_LIST) + client.emit(message.message, message.payload...) + cb() + , (err) -> + if err? + logger.err {err, message}, "Error sending message to clients" else if message.health_check? logger.debug {message}, "got health check message in editor events channel" HealthCheckManager.check channel, message.key - diff --git a/services/real-time/test/unit/coffee/WebApiManagerTests.coffee b/services/real-time/test/unit/coffee/WebApiManagerTests.coffee index 278585518d..a87522c0a9 100644 --- a/services/real-time/test/unit/coffee/WebApiManagerTests.coffee +++ b/services/real-time/test/unit/coffee/WebApiManagerTests.coffee @@ -20,17 +20,18 @@ describe 'WebApiManager', -> user: "username" pass: "password" "logger-sharelatex": @logger = { log: sinon.stub(), error: sinon.stub() } - + describe "joinProject", -> describe "successfully", -> beforeEach -> @response = { project: { name: "Test project" } - privilegeLevel: "owner" + privilegeLevel: "owner", + isRestrictedUser: true } @request.post = sinon.stub().callsArgWith(1, null, {statusCode: 200}, @response) @WebApiManager.joinProject @project_id, @user, @callback - + it "should send a request to web to join the project", -> @request.post .calledWith({ @@ -46,22 +47,32 @@ describe 'WebApiManager', -> headers: {} }) .should.equal true - - it "should return the project and privilegeLevel", -> + + it "should return the project, privilegeLevel, and restricted flag", -> @callback - .calledWith(null, @response.project, @response.privilegeLevel) + .calledWith(null, @response.project, @response.privilegeLevel, @response.isRestrictedUser) .should.equal true - + describe "with an error from web", -> beforeEach -> @request.post = sinon.stub().callsArgWith(1, null, {statusCode: 500}, null) @WebApiManager.joinProject @project_id, @user_id, @callback - + it "should call the callback with an error", -> @callback .calledWith(new Error("non-success code from web: 500")) .should.equal true - + + describe "with no data from web", -> + beforeEach -> + @request.post = sinon.stub().callsArgWith(1, null, {statusCode: 200}, null) + @WebApiManager.joinProject @project_id, @user_id, @callback + + it "should call the callback with an error", -> + @callback + .calledWith(new Error("no data returned from joinProject request")) + .should.equal true + describe "when the project is over its rate limit", -> beforeEach -> @request.post = sinon.stub().callsArgWith(1, null, {statusCode: 429}, null) diff --git a/services/real-time/test/unit/coffee/WebsocketControllerTests.coffee b/services/real-time/test/unit/coffee/WebsocketControllerTests.coffee index edb0055bf5..116485384d 100644 --- a/services/real-time/test/unit/coffee/WebsocketControllerTests.coffee +++ b/services/real-time/test/unit/coffee/WebsocketControllerTests.coffee @@ -37,10 +37,10 @@ describe 'WebsocketController', -> inc: sinon.stub() set: sinon.stub() "./RoomManager": @RoomManager = {} - + afterEach -> tk.reset() - + describe "joinProject", -> describe "when authorised", -> beforeEach -> @@ -53,61 +53,65 @@ describe 'WebsocketController', -> } @privilegeLevel = "owner" @ConnectedUsersManager.updateUserPosition = sinon.stub().callsArg(4) - @WebApiManager.joinProject = sinon.stub().callsArgWith(2, null, @project, @privilegeLevel) + @isRestrictedUser = true + @WebApiManager.joinProject = sinon.stub().callsArgWith(2, null, @project, @privilegeLevel, @isRestrictedUser) @RoomManager.joinProject = sinon.stub().callsArg(2) @WebsocketController.joinProject @client, @user, @project_id, @callback - + it "should load the project from web", -> @WebApiManager.joinProject .calledWith(@project_id, @user) .should.equal true - + it "should join the project room", -> @RoomManager.joinProject.calledWith(@client, @project_id).should.equal true - + it "should set the privilege level on the client", -> @client.set.calledWith("privilege_level", @privilegeLevel).should.equal true - + it "should set the user's id on the client", -> @client.set.calledWith("user_id", @user._id).should.equal true - + it "should set the user's email on the client", -> @client.set.calledWith("email", @user.email).should.equal true - + it "should set the user's first_name on the client", -> @client.set.calledWith("first_name", @user.first_name).should.equal true - + it "should set the user's last_name on the client", -> @client.set.calledWith("last_name", @user.last_name).should.equal true - + it "should set the user's sign up date on the client", -> @client.set.calledWith("signup_date", @user.signUpDate).should.equal true - + it "should set the user's login_count on the client", -> @client.set.calledWith("login_count", @user.loginCount).should.equal true - + it "should set the connected time on the client", -> @client.set.calledWith("connected_time", new Date()).should.equal true - + it "should set the project_id on the client", -> @client.set.calledWith("project_id", @project_id).should.equal true - + it "should set the project owner id on the client", -> @client.set.calledWith("owner_id", @owner_id).should.equal true - + + it "should set the is_restricted_user flag on the client", -> + @client.set.calledWith("is_restricted_user", @isRestrictedUser).should.equal true + it "should call the callback with the project, privilegeLevel and protocolVersion", -> @callback .calledWith(null, @project, @privilegeLevel, @WebsocketController.PROTOCOL_VERSION) .should.equal true - + it "should mark the user as connected in ConnectedUsersManager", -> @ConnectedUsersManager.updateUserPosition .calledWith(@project_id, @client.id, @user, null) .should.equal true - + it "should increment the join-project metric", -> @metrics.inc.calledWith("editor.join-project").should.equal true - + describe "when not authorized", -> beforeEach -> @WebApiManager.joinProject = sinon.stub().callsArgWith(2, null, null, null) @@ -138,30 +142,30 @@ describe 'WebsocketController', -> @client.params.user_id = @user_id @WebsocketController.FLUSH_IF_EMPTY_DELAY = 0 tk.reset() # Allow setTimeout to work. - + describe "when the project is empty", -> beforeEach (done) -> @clientsInRoom = [] @WebsocketController.leaveProject @io, @client, done - + it "should end clientTracking.clientDisconnected to the project room", -> @WebsocketLoadBalancer.emitToRoom .calledWith(@project_id, "clientTracking.clientDisconnected", @client.id) .should.equal true - + it "should mark the user as disconnected", -> @ConnectedUsersManager.markUserAsDisconnected .calledWith(@project_id, @client.id) .should.equal true - + it "should flush the project in the document updater", -> @DocumentUpdaterManager.flushProjectToMongoAndDelete .calledWith(@project_id) .should.equal true - + it "should increment the leave-project metric", -> @metrics.inc.calledWith("editor.leave-project").should.equal true - + it "should track the disconnection in RoomManager", -> @RoomManager.leaveProjectAndDocs .calledWith(@client) @@ -171,7 +175,7 @@ describe 'WebsocketController', -> beforeEach -> @clientsInRoom = ["mock-remaining-client"] @WebsocketController.leaveProject @io, @client - + it "should not flush the project in the document updater", -> @DocumentUpdaterManager.flushProjectToMongoAndDelete .called.should.equal false @@ -232,7 +236,7 @@ describe 'WebsocketController', -> @ops = ["mock", "ops"] @ranges = { "mock": "ranges" } @options = {} - + @client.params.project_id = @project_id @AuthorizationManager.addAccessToDoc = sinon.stub() @AuthorizationManager.assertClientCanViewProject = sinon.stub().callsArgWith(1, null) @@ -275,7 +279,7 @@ describe 'WebsocketController', -> beforeEach -> @fromVersion = 40 @WebsocketController.joinDoc @client, @doc_id, @fromVersion, @options, @callback - + it "should get the document from the DocumentUpdaterManager with fromVersion", -> @DocumentUpdaterManager.getDocument .calledWith(@project_id, @doc_id, @fromVersion) @@ -285,7 +289,7 @@ describe 'WebsocketController', -> beforeEach -> @doc_lines.push ["räksmörgås"] @WebsocketController.joinDoc @client, @doc_id, -1, @options, @callback - + it "should call the callback with the escaped lines", -> escaped_lines = @callback.args[0][1] escaped_word = escaped_lines.pop() @@ -327,44 +331,44 @@ describe 'WebsocketController', -> beforeEach -> @AuthorizationManager.assertClientCanViewProject = sinon.stub().callsArgWith(1, @err = new Error("not authorized")) @WebsocketController.joinDoc @client, @doc_id, -1, @options, @callback - + it "should call the callback with an error", -> @callback.calledWith(@err).should.equal true - + it "should not call the DocumentUpdaterManager", -> @DocumentUpdaterManager.getDocument.called.should.equal false - + describe "leaveDoc", -> beforeEach -> - @doc_id = "doc-id-123" + @doc_id = "doc-id-123" @client.params.project_id = @project_id @RoomManager.leaveDoc = sinon.stub() @WebsocketController.leaveDoc @client, @doc_id, @callback - + it "should remove the client from the doc_id room", -> @RoomManager.leaveDoc .calledWith(@client, @doc_id).should.equal true - + it "should call the callback", -> @callback.called.should.equal true - + it "should increment the leave-doc metric", -> @metrics.inc.calledWith("editor.leave-doc").should.equal true - + describe "getConnectedUsers", -> beforeEach -> @client.params.project_id = @project_id @users = ["mock", "users"] @WebsocketLoadBalancer.emitToRoom = sinon.stub() @ConnectedUsersManager.getConnectedUsers = sinon.stub().callsArgWith(1, null, @users) - + describe "when authorized", -> beforeEach (done) -> @AuthorizationManager.assertClientCanViewProject = sinon.stub().callsArgWith(1, null) @WebsocketController.getConnectedUsers @client, (args...) => @callback(args...) done() - + it "should check that the client is authorized to view the project", -> @AuthorizationManager.assertClientCanViewProject .calledWith(@client) @@ -379,26 +383,40 @@ describe 'WebsocketController', -> @ConnectedUsersManager.getConnectedUsers .calledWith(@project_id) .should.equal true - + it "should return the users", -> @callback.calledWith(null, @users).should.equal true - + it "should increment the get-connected-users metric", -> @metrics.inc.calledWith("editor.get-connected-users").should.equal true - + describe "when not authorized", -> beforeEach -> @AuthorizationManager.assertClientCanViewProject = sinon.stub().callsArgWith(1, @err = new Error("not authorized")) @WebsocketController.getConnectedUsers @client, @callback - + it "should not get the connected users for the project", -> @ConnectedUsersManager.getConnectedUsers .called .should.equal false - + it "should return an error", -> @callback.calledWith(@err).should.equal true - + + describe "when restricted user", -> + beforeEach -> + @client.params.is_restricted_user = true + @AuthorizationManager.assertClientCanViewProject = sinon.stub().callsArgWith(1, null) + @WebsocketController.getConnectedUsers @client, @callback + + it "should return an empty array of users", -> + @callback.calledWith(null, []).should.equal true + + it "should not get the connected users for the project", -> + @ConnectedUsersManager.getConnectedUsers + .called + .should.equal false + describe "updateClientPosition", -> beforeEach -> @WebsocketLoadBalancer.emitToRoom = sinon.stub() @@ -422,7 +440,7 @@ describe 'WebsocketController', -> @client.get = (param, callback) => callback null, @clientParams[param] @WebsocketController.updateClientPosition @client, @update - @populatedCursorData = + @populatedCursorData = doc_id: @doc_id, id: @client.id name: "#{@first_name} #{@last_name}" @@ -462,7 +480,7 @@ describe 'WebsocketController', -> @client.get = (param, callback) => callback null, @clientParams[param] @WebsocketController.updateClientPosition @client, @update - @populatedCursorData = + @populatedCursorData = doc_id: @doc_id, id: @client.id name: "#{@first_name}" @@ -502,7 +520,7 @@ describe 'WebsocketController', -> @client.get = (param, callback) => callback null, @clientParams[param] @WebsocketController.updateClientPosition @client, @update - @populatedCursorData = + @populatedCursorData = doc_id: @doc_id, id: @client.id name: "#{@last_name}" @@ -603,7 +621,7 @@ describe 'WebsocketController', -> it "should call the callback", -> @callback.called.should.equal true - + it "should increment the doc updates", -> @metrics.inc.calledWith("editor.doc-update").should.equal true @@ -621,7 +639,7 @@ describe 'WebsocketController', -> it "should call the callback with the error", -> @callback.calledWith(@error).should.equal true - + describe "when not authorized", -> beforeEach -> @client.disconnect = sinon.stub() @@ -645,7 +663,7 @@ describe 'WebsocketController', -> @comment_update = { op: [{c: "bar", p: 132}] } @AuthorizationManager.assertClientCanEditProjectAndDoc = sinon.stub() @AuthorizationManager.assertClientCanViewProjectAndDoc = sinon.stub() - + describe "with a read-write client", -> it "should return successfully", (done) -> @AuthorizationManager.assertClientCanEditProjectAndDoc.yields(null) @@ -660,7 +678,7 @@ describe 'WebsocketController', -> @WebsocketController._assertClientCanApplyUpdate @client, @doc_id, @edit_update, (error) -> expect(error.message).to.equal "not authorized" done() - + describe "with a read-only client and a comment op", -> it "should return successfully", (done) -> @AuthorizationManager.assertClientCanEditProjectAndDoc.yields(new Error("not authorized")) @@ -668,7 +686,7 @@ describe 'WebsocketController', -> @WebsocketController._assertClientCanApplyUpdate @client, @doc_id, @comment_update, (error) -> expect(error).to.be.null done() - + describe "with a totally unauthorized client", -> it "should return an error", (done) -> @AuthorizationManager.assertClientCanEditProjectAndDoc.yields(new Error("not authorized")) diff --git a/services/real-time/test/unit/coffee/WebsocketLoadBalancerTests.coffee b/services/real-time/test/unit/coffee/WebsocketLoadBalancerTests.coffee index 66e950cd0e..e6fe1df8c9 100644 --- a/services/real-time/test/unit/coffee/WebsocketLoadBalancerTests.coffee +++ b/services/real-time/test/unit/coffee/WebsocketLoadBalancerTests.coffee @@ -8,7 +8,7 @@ describe "WebsocketLoadBalancer", -> @rclient = {} @RoomEvents = {on: sinon.stub()} @WebsocketLoadBalancer = SandboxedModule.require modulePath, requires: - "./RedisClientManager": + "./RedisClientManager": createClientList: () => [] "logger-sharelatex": @logger = { log: sinon.stub(), error: sinon.stub() } "./SafeJsonParse": @SafeJsonParse = @@ -18,6 +18,12 @@ describe "WebsocketLoadBalancer", -> "./RoomManager" : @RoomManager = {eventSource: sinon.stub().returns @RoomEvents} "./ChannelManager": @ChannelManager = {publish: sinon.stub()} "./ConnectedUsersManager": @ConnectedUsersManager = {refreshClient: sinon.stub()} + "./Utils": @Utils = { + getClientAttributes: sinon.spy( + (client, _attrs, callback) -> + callback(null, {is_restricted_user: !!client.__isRestricted}) + ) + } @io = {} @WebsocketLoadBalancer.rclientPubList = [{publish: sinon.stub()}] @WebsocketLoadBalancer.rclientSubList = [{ @@ -26,7 +32,7 @@ describe "WebsocketLoadBalancer", -> }] @room_id = "room-id" - @message = "message-to-editor" + @message = "otUpdateApplied" @payload = ["argument one", 42] describe "emitToRoom", -> @@ -51,7 +57,7 @@ describe "WebsocketLoadBalancer", -> @WebsocketLoadBalancer.emitToRoom .calledWith("all", @message, @payload...) .should.equal true - + describe "listenForEditorEvents", -> beforeEach -> @WebsocketLoadBalancer._processEditorEvent = sinon.stub() @@ -70,9 +76,10 @@ describe "WebsocketLoadBalancer", -> describe "_processEditorEvent", -> describe "with bad JSON", -> beforeEach -> + @isRestrictedUser = false @SafeJsonParse.parse = sinon.stub().callsArgWith 1, new Error("oops") @WebsocketLoadBalancer._processEditorEvent(@io, "editor-events", "blah") - + it "should log an error", -> @logger.error.called.should.equal true @@ -98,6 +105,54 @@ describe "WebsocketLoadBalancer", -> @emit2.calledWith(@message, @payload...).should.equal true @emit3.called.should.equal false # duplicate client should be ignored + describe "with a designated room, and restricted clients, not restricted message", -> + beforeEach -> + @io.sockets = + clients: sinon.stub().returns([ + {id: 'client-id-1', emit: @emit1 = sinon.stub()} + {id: 'client-id-2', emit: @emit2 = sinon.stub()} + {id: 'client-id-1', emit: @emit3 = sinon.stub()} # duplicate client + {id: 'client-id-4', emit: @emit4 = sinon.stub(), __isRestricted: true} + ]) + data = JSON.stringify + room_id: @room_id + message: @message + payload: @payload + @WebsocketLoadBalancer._processEditorEvent(@io, "editor-events", data) + + it "should send the message to all (unique) clients in the room", -> + @io.sockets.clients + .calledWith(@room_id) + .should.equal true + @emit1.calledWith(@message, @payload...).should.equal true + @emit2.calledWith(@message, @payload...).should.equal true + @emit3.called.should.equal false # duplicate client should be ignored + @emit4.called.should.equal true # restricted client, but should be called + + describe "with a designated room, and restricted clients, restricted message", -> + beforeEach -> + @io.sockets = + clients: sinon.stub().returns([ + {id: 'client-id-1', emit: @emit1 = sinon.stub()} + {id: 'client-id-2', emit: @emit2 = sinon.stub()} + {id: 'client-id-1', emit: @emit3 = sinon.stub()} # duplicate client + {id: 'client-id-4', emit: @emit4 = sinon.stub(), __isRestricted: true} + ]) + data = JSON.stringify + room_id: @room_id + message: @restrictedMessage = 'new-comment' + payload: @payload + @WebsocketLoadBalancer._processEditorEvent(@io, "editor-events", data) + + it "should send the message to all (unique) clients in the room, who are not restricted", -> + @io.sockets.clients + .calledWith(@room_id) + .should.equal true + @emit1.calledWith(@restrictedMessage, @payload...).should.equal true + @emit2.calledWith(@restrictedMessage, @payload...).should.equal true + @emit3.called.should.equal false # duplicate client should be ignored + @emit4.called.should.equal false # restricted client, should not be called + describe "when emitting to all", -> beforeEach -> @io.sockets = @@ -110,4 +165,3 @@ describe "WebsocketLoadBalancer", -> it "should send the message to all clients", -> @emit.calledWith(@message, @payload...).should.equal true - From 6df88ebc49679e4683211e00f48e6bbd37b82a67 Mon Sep 17 00:00:00 2001 From: Shane Kilkelly Date: Wed, 30 Oct 2019 10:15:20 +0000 Subject: [PATCH 299/491] Filter "comments" if restricted user. --- .../real-time/app/coffee/WebsocketController.coffee | 5 ++++- .../test/unit/coffee/WebsocketControllerTests.coffee | 11 +++++++++++ 2 files changed, 15 insertions(+), 1 deletion(-) diff --git a/services/real-time/app/coffee/WebsocketController.coffee b/services/real-time/app/coffee/WebsocketController.coffee index d0ca99cc5c..d091c5e4ed 100644 --- a/services/real-time/app/coffee/WebsocketController.coffee +++ b/services/real-time/app/coffee/WebsocketController.coffee @@ -85,7 +85,7 @@ module.exports = WebsocketController = joinDoc: (client, doc_id, fromVersion = -1, options, callback = (error, doclines, version, ops, ranges) ->) -> metrics.inc "editor.join-doc" - Utils.getClientAttributes client, ["project_id", "user_id"], (error, {project_id, user_id}) -> + Utils.getClientAttributes client, ["project_id", "user_id", "is_restricted_user"], (error, {project_id, user_id, is_restricted_user}) -> return callback(error) if error? return callback(new Error("no project_id found on client")) if !project_id? logger.log {user_id, project_id, doc_id, fromVersion, client_id: client.id}, "client joining doc" @@ -99,6 +99,9 @@ module.exports = WebsocketController = DocumentUpdaterManager.getDocument project_id, doc_id, fromVersion, (error, lines, version, ranges, ops) -> return callback(error) if error? + if is_restricted_user and ranges?.comments? + ranges.comments = [] + # Encode any binary bits of data so it can go via WebSockets # See http://ecmanaut.blogspot.co.uk/2006/07/encoding-decoding-utf8-in-javascript.html encodeForWebsockets = (text) -> unescape(encodeURIComponent(text)) diff --git a/services/real-time/test/unit/coffee/WebsocketControllerTests.coffee b/services/real-time/test/unit/coffee/WebsocketControllerTests.coffee index 116485384d..f2f1531834 100644 --- a/services/real-time/test/unit/coffee/WebsocketControllerTests.coffee +++ b/services/real-time/test/unit/coffee/WebsocketControllerTests.coffee @@ -238,6 +238,7 @@ describe 'WebsocketController', -> @options = {} @client.params.project_id = @project_id + @client.params.is_restricted_user = false @AuthorizationManager.addAccessToDoc = sinon.stub() @AuthorizationManager.assertClientCanViewProject = sinon.stub().callsArgWith(1, null) @DocumentUpdaterManager.getDocument = sinon.stub().callsArgWith(3, null, @doc_lines, @version, @ranges, @ops) @@ -338,6 +339,16 @@ describe 'WebsocketController', -> it "should not call the DocumentUpdaterManager", -> @DocumentUpdaterManager.getDocument.called.should.equal false + describe "with a restricted client", -> + beforeEach -> + @ranges.comments = [{op: {a: 1}}, {op: {a: 2}}] + @client.params.is_restricted_user = true + @WebsocketController.joinDoc @client, @doc_id, -1, @options, @callback + + it "should overwrite ranges.comments with an empty list", -> + ranges = @callback.args[0][4] + expect(ranges.comments).to.deep.equal [] + describe "leaveDoc", -> beforeEach -> @doc_id = "doc-id-123" From e04b6e1e49ca7710afcdd7e28581103f9e59ed4d Mon Sep 17 00:00:00 2001 From: Brian Gough Date: Mon, 3 Feb 2020 14:46:14 +0000 Subject: [PATCH 300/491] Update app/coffee/Router.coffee Co-Authored-By: Jakob Ackermann --- services/real-time/app/coffee/Router.coffee | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/services/real-time/app/coffee/Router.coffee b/services/real-time/app/coffee/Router.coffee index ae4d404e48..0ec305ed3a 100644 --- a/services/real-time/app/coffee/Router.coffee +++ b/services/real-time/app/coffee/Router.coffee @@ -43,7 +43,7 @@ module.exports = Router = session.on 'connection', (error, client, session) -> client?.on "error", (err) -> - logger.err "socket.io client error", { err } + logger.err { err }, "socket.io client error" if client.connected client.emit("reconnectGracefully") client.disconnect() From 49a8e1214b476eeae2e80d3a6fc442759c20bc8b Mon Sep 17 00:00:00 2001 From: Brian Gough Date: Mon, 3 Feb 2020 14:47:45 +0000 Subject: [PATCH 301/491] use a separate field for client errors --- services/real-time/app/coffee/Router.coffee | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/services/real-time/app/coffee/Router.coffee b/services/real-time/app/coffee/Router.coffee index 0ec305ed3a..fcdb0c93dc 100644 --- a/services/real-time/app/coffee/Router.coffee +++ b/services/real-time/app/coffee/Router.coffee @@ -43,7 +43,7 @@ module.exports = Router = session.on 'connection', (error, client, session) -> client?.on "error", (err) -> - logger.err { err }, "socket.io client error" + logger.err { clientErr: err }, "socket.io client error" if client.connected client.emit("reconnectGracefully") client.disconnect() From ef852dfa3363125d54f6a5f5bb906b127d5ac860 Mon Sep 17 00:00:00 2001 From: Simon Detheridge Date: Tue, 4 Feb 2020 10:32:54 +0000 Subject: [PATCH 302/491] Update socket.io to latest patch release --- services/real-time/npm-shrinkwrap.json | 2039 ++++++++++++++++-------- services/real-time/package.json | 2 +- 2 files changed, 1398 insertions(+), 643 deletions(-) diff --git a/services/real-time/npm-shrinkwrap.json b/services/real-time/npm-shrinkwrap.json index bc7bba57e5..3b393f9278 100644 --- a/services/real-time/npm-shrinkwrap.json +++ b/services/real-time/npm-shrinkwrap.json @@ -1,1897 +1,2652 @@ { "name": "real-time-sharelatex", "version": "0.1.4", + "lockfileVersion": 1, + "requires": true, "dependencies": { "@google-cloud/common": { "version": "0.32.1", - "from": "@google-cloud/common@>=0.32.0 <0.33.0", - "resolved": "https://registry.npmjs.org/@google-cloud/common/-/common-0.32.1.tgz" + "resolved": "https://registry.npmjs.org/@google-cloud/common/-/common-0.32.1.tgz", + "integrity": "sha1-ajLDQBcs6j22Z00ODjTnh0CgBz8=", + "requires": { + "@google-cloud/projectify": "^0.3.3", + "@google-cloud/promisify": "^0.4.0", + "@types/request": "^2.48.1", + "arrify": "^2.0.0", + "duplexify": "^3.6.0", + "ent": "^2.2.0", + "extend": "^3.0.2", + "google-auth-library": "^3.1.1", + "pify": "^4.0.1", + "retry-request": "^4.0.0", + "teeny-request": "^3.11.3" + } }, "@google-cloud/debug-agent": { "version": "3.2.0", - "from": "@google-cloud/debug-agent@>=3.0.0 <4.0.0", "resolved": "https://registry.npmjs.org/@google-cloud/debug-agent/-/debug-agent-3.2.0.tgz", + "integrity": "sha1-2qdjWhaYpWY31dxXzhED536uKdM=", + "requires": { + "@google-cloud/common": "^0.32.0", + "@sindresorhus/is": "^0.15.0", + "acorn": "^6.0.0", + "coffeescript": "^2.0.0", + "console-log-level": "^1.4.0", + "extend": "^3.0.1", + "findit2": "^2.2.3", + "gcp-metadata": "^1.0.0", + "lodash.pickby": "^4.6.0", + "p-limit": "^2.2.0", + "pify": "^4.0.1", + "semver": "^6.0.0", + "source-map": "^0.6.1", + "split": "^1.0.0" + }, "dependencies": { "coffeescript": { "version": "2.4.1", - "from": "coffeescript@>=2.0.0 <3.0.0", - "resolved": "https://registry.npmjs.org/coffeescript/-/coffeescript-2.4.1.tgz" + "resolved": "https://registry.npmjs.org/coffeescript/-/coffeescript-2.4.1.tgz", + "integrity": "sha1-gV/TN98KNNSedKmKbr6pw+eTD3A=" } } }, "@google-cloud/profiler": { "version": "0.2.3", - "from": "@google-cloud/profiler@>=0.2.3 <0.3.0", "resolved": "https://registry.npmjs.org/@google-cloud/profiler/-/profiler-0.2.3.tgz", + "integrity": "sha1-Fj3738Mwuug1X+RuHlvgZTV7H1w=", + "requires": { + "@google-cloud/common": "^0.26.0", + "@types/console-log-level": "^1.4.0", + "@types/semver": "^5.5.0", + "bindings": "^1.2.1", + "console-log-level": "^1.4.0", + "delay": "^4.0.1", + "extend": "^3.0.1", + "gcp-metadata": "^0.9.0", + "nan": "^2.11.1", + "parse-duration": "^0.1.1", + "pify": "^4.0.0", + "pretty-ms": "^4.0.0", + "protobufjs": "~6.8.6", + "semver": "^5.5.0", + "teeny-request": "^3.3.0" + }, "dependencies": { "@google-cloud/common": { "version": "0.26.2", - "from": "@google-cloud/common@>=0.26.0 <0.27.0", - "resolved": "https://registry.npmjs.org/@google-cloud/common/-/common-0.26.2.tgz" + "resolved": "https://registry.npmjs.org/@google-cloud/common/-/common-0.26.2.tgz", + "integrity": "sha1-nFTiRxqEqgMelaJIJJduCA8lVkU=", + "requires": { + "@google-cloud/projectify": "^0.3.2", + "@google-cloud/promisify": "^0.3.0", + "@types/duplexify": "^3.5.0", + "@types/request": "^2.47.0", + "arrify": "^1.0.1", + "duplexify": "^3.6.0", + "ent": "^2.2.0", + "extend": "^3.0.1", + "google-auth-library": "^2.0.0", + "pify": "^4.0.0", + "retry-request": "^4.0.0", + "through2": "^3.0.0" + } }, "@google-cloud/promisify": { "version": "0.3.1", - "from": "@google-cloud/promisify@>=0.3.0 <0.4.0", - "resolved": "https://registry.npmjs.org/@google-cloud/promisify/-/promisify-0.3.1.tgz" + "resolved": "https://registry.npmjs.org/@google-cloud/promisify/-/promisify-0.3.1.tgz", + "integrity": "sha1-9kHm2USo4KBe4MsQkd+mAIm+zbo=" }, "arrify": { "version": "1.0.1", - "from": "arrify@>=1.0.1 <2.0.0", - "resolved": "https://registry.npmjs.org/arrify/-/arrify-1.0.1.tgz" + "resolved": "https://registry.npmjs.org/arrify/-/arrify-1.0.1.tgz", + "integrity": "sha1-iYUI2iIm84DfkEcoRWhJwVAaSw0=" }, "gcp-metadata": { "version": "0.9.3", - "from": "gcp-metadata@>=0.9.0 <0.10.0", - "resolved": "https://registry.npmjs.org/gcp-metadata/-/gcp-metadata-0.9.3.tgz" + "resolved": "https://registry.npmjs.org/gcp-metadata/-/gcp-metadata-0.9.3.tgz", + "integrity": "sha1-H510lfdGChRSZIHynhFZbdVj3SY=", + "requires": { + "gaxios": "^1.0.2", + "json-bigint": "^0.3.0" + } }, "google-auth-library": { "version": "2.0.2", - "from": "google-auth-library@>=2.0.0 <3.0.0", "resolved": "https://registry.npmjs.org/google-auth-library/-/google-auth-library-2.0.2.tgz", + "integrity": "sha1-ejFdIDZ0Svavyth7IQ7mY4tA9Xs=", + "requires": { + "axios": "^0.18.0", + "gcp-metadata": "^0.7.0", + "gtoken": "^2.3.0", + "https-proxy-agent": "^2.2.1", + "jws": "^3.1.5", + "lru-cache": "^5.0.0", + "semver": "^5.5.0" + }, "dependencies": { "gcp-metadata": { "version": "0.7.0", - "from": "gcp-metadata@>=0.7.0 <0.8.0", - "resolved": "https://registry.npmjs.org/gcp-metadata/-/gcp-metadata-0.7.0.tgz" + "resolved": "https://registry.npmjs.org/gcp-metadata/-/gcp-metadata-0.7.0.tgz", + "integrity": "sha1-bDXbtSvaMqQnu5yY9UI33dG1QG8=", + "requires": { + "axios": "^0.18.0", + "extend": "^3.0.1", + "retry-axios": "0.3.2" + } } } }, "semver": { "version": "5.7.0", - "from": "semver@>=5.5.0 <6.0.0", - "resolved": "https://registry.npmjs.org/semver/-/semver-5.7.0.tgz" + "resolved": "https://registry.npmjs.org/semver/-/semver-5.7.0.tgz", + "integrity": "sha1-eQp89v6lRZuslhELKbYEEtyP+Ws=" }, "through2": { "version": "3.0.1", - "from": "through2@>=3.0.0 <4.0.0", - "resolved": "https://registry.npmjs.org/through2/-/through2-3.0.1.tgz" + "resolved": "https://registry.npmjs.org/through2/-/through2-3.0.1.tgz", + "integrity": "sha1-OSducTwzAu3544jdnIEt07glvVo=", + "requires": { + "readable-stream": "2 || 3" + } } } }, "@google-cloud/projectify": { "version": "0.3.3", - "from": "@google-cloud/projectify@>=0.3.3 <0.4.0", - "resolved": "https://registry.npmjs.org/@google-cloud/projectify/-/projectify-0.3.3.tgz" + "resolved": "https://registry.npmjs.org/@google-cloud/projectify/-/projectify-0.3.3.tgz", + "integrity": "sha1-vekQPVCyCj6jM334xng6dm5w1B0=" }, "@google-cloud/promisify": { "version": "0.4.0", - "from": "@google-cloud/promisify@>=0.4.0 <0.5.0", - "resolved": "https://registry.npmjs.org/@google-cloud/promisify/-/promisify-0.4.0.tgz" + "resolved": "https://registry.npmjs.org/@google-cloud/promisify/-/promisify-0.4.0.tgz", + "integrity": "sha1-T7/PTYW7ai5MzwWqY9KxDWyarZs=" }, "@google-cloud/trace-agent": { "version": "3.6.1", - "from": "@google-cloud/trace-agent@>=3.2.0 <4.0.0", "resolved": "https://registry.npmjs.org/@google-cloud/trace-agent/-/trace-agent-3.6.1.tgz", + "integrity": "sha1-W+dEE5TQ6ldY8o25IqUAT/PwO+w=", + "requires": { + "@google-cloud/common": "^0.32.1", + "builtin-modules": "^3.0.0", + "console-log-level": "^1.4.0", + "continuation-local-storage": "^3.2.1", + "extend": "^3.0.0", + "gcp-metadata": "^1.0.0", + "hex2dec": "^1.0.1", + "is": "^3.2.0", + "methods": "^1.1.1", + "require-in-the-middle": "^4.0.0", + "semver": "^6.0.0", + "shimmer": "^1.2.0", + "uuid": "^3.0.1" + }, "dependencies": { "uuid": { "version": "3.3.2", - "from": "uuid@^3.0.1", - "resolved": "https://registry.npmjs.org/uuid/-/uuid-3.3.2.tgz" + "resolved": "https://registry.npmjs.org/uuid/-/uuid-3.3.2.tgz", + "integrity": "sha1-G0r0lV6zB3xQHCOHL8ZROBFYcTE=" } } }, "@protobufjs/aspromise": { "version": "1.1.2", - "from": "@protobufjs/aspromise@>=1.1.2 <2.0.0", - "resolved": "https://registry.npmjs.org/@protobufjs/aspromise/-/aspromise-1.1.2.tgz" + "resolved": "https://registry.npmjs.org/@protobufjs/aspromise/-/aspromise-1.1.2.tgz", + "integrity": "sha1-m4sMxmPWaafY9vXQiToU00jzD78=" }, "@protobufjs/base64": { "version": "1.1.2", - "from": "@protobufjs/base64@>=1.1.2 <2.0.0", - "resolved": "https://registry.npmjs.org/@protobufjs/base64/-/base64-1.1.2.tgz" + "resolved": "https://registry.npmjs.org/@protobufjs/base64/-/base64-1.1.2.tgz", + "integrity": "sha1-TIVzDlm5ofHzSQR9vyQpYDS7JzU=" }, "@protobufjs/codegen": { "version": "2.0.4", - "from": "@protobufjs/codegen@>=2.0.4 <3.0.0", - "resolved": "https://registry.npmjs.org/@protobufjs/codegen/-/codegen-2.0.4.tgz" + "resolved": "https://registry.npmjs.org/@protobufjs/codegen/-/codegen-2.0.4.tgz", + "integrity": "sha1-fvN/DQEPsCitGtWXIuUG2SYoFcs=" }, "@protobufjs/eventemitter": { "version": "1.1.0", - "from": "@protobufjs/eventemitter@>=1.1.0 <2.0.0", - "resolved": "https://registry.npmjs.org/@protobufjs/eventemitter/-/eventemitter-1.1.0.tgz" + "resolved": "https://registry.npmjs.org/@protobufjs/eventemitter/-/eventemitter-1.1.0.tgz", + "integrity": "sha1-NVy8mLr61ZePntCV85diHx0Ga3A=" }, "@protobufjs/fetch": { "version": "1.1.0", - "from": "@protobufjs/fetch@>=1.1.0 <2.0.0", - "resolved": "https://registry.npmjs.org/@protobufjs/fetch/-/fetch-1.1.0.tgz" + "resolved": "https://registry.npmjs.org/@protobufjs/fetch/-/fetch-1.1.0.tgz", + "integrity": "sha1-upn7WYYUr2VwDBYZ/wbUVLDYTEU=", + "requires": { + "@protobufjs/aspromise": "^1.1.1", + "@protobufjs/inquire": "^1.1.0" + } }, "@protobufjs/float": { "version": "1.0.2", - "from": "@protobufjs/float@>=1.0.2 <2.0.0", - "resolved": "https://registry.npmjs.org/@protobufjs/float/-/float-1.0.2.tgz" + "resolved": "https://registry.npmjs.org/@protobufjs/float/-/float-1.0.2.tgz", + "integrity": "sha1-Xp4avctz/Ap8uLKR33jIy9l7h9E=" }, "@protobufjs/inquire": { "version": "1.1.0", - "from": "@protobufjs/inquire@>=1.1.0 <2.0.0", - "resolved": "https://registry.npmjs.org/@protobufjs/inquire/-/inquire-1.1.0.tgz" + "resolved": "https://registry.npmjs.org/@protobufjs/inquire/-/inquire-1.1.0.tgz", + "integrity": "sha1-/yAOPnzyQp4tyvwRQIKOjMY48Ik=" }, "@protobufjs/path": { "version": "1.1.2", - "from": "@protobufjs/path@>=1.1.2 <2.0.0", - "resolved": "https://registry.npmjs.org/@protobufjs/path/-/path-1.1.2.tgz" + "resolved": "https://registry.npmjs.org/@protobufjs/path/-/path-1.1.2.tgz", + "integrity": "sha1-bMKyDFya1q0NzP0hynZz2Nf79o0=" }, "@protobufjs/pool": { "version": "1.1.0", - "from": "@protobufjs/pool@>=1.1.0 <2.0.0", - "resolved": "https://registry.npmjs.org/@protobufjs/pool/-/pool-1.1.0.tgz" + "resolved": "https://registry.npmjs.org/@protobufjs/pool/-/pool-1.1.0.tgz", + "integrity": "sha1-Cf0V8tbTq/qbZbw2ZQbWrXhG/1Q=" }, "@protobufjs/utf8": { "version": "1.1.0", - "from": "@protobufjs/utf8@>=1.1.0 <2.0.0", - "resolved": "https://registry.npmjs.org/@protobufjs/utf8/-/utf8-1.1.0.tgz" + "resolved": "https://registry.npmjs.org/@protobufjs/utf8/-/utf8-1.1.0.tgz", + "integrity": "sha1-p3c2C1s5oaLlEG+OhY8v0tBgxXA=" }, "@sindresorhus/is": { "version": "0.15.0", - "from": "@sindresorhus/is@>=0.15.0 <0.16.0", - "resolved": "https://registry.npmjs.org/@sindresorhus/is/-/is-0.15.0.tgz" + "resolved": "https://registry.npmjs.org/@sindresorhus/is/-/is-0.15.0.tgz", + "integrity": "sha1-lpFbqgXmpqHRN7rfSYTT/AWCC7Y=" }, "@types/caseless": { "version": "0.12.2", - "from": "@types/caseless@*", - "resolved": "https://registry.npmjs.org/@types/caseless/-/caseless-0.12.2.tgz" + "resolved": "https://registry.npmjs.org/@types/caseless/-/caseless-0.12.2.tgz", + "integrity": "sha1-9l09Y4ngHutFi9VNyPUrlalGO8g=" }, "@types/console-log-level": { "version": "1.4.0", - "from": "@types/console-log-level@>=1.4.0 <2.0.0", - "resolved": "https://registry.npmjs.org/@types/console-log-level/-/console-log-level-1.4.0.tgz" + "resolved": "https://registry.npmjs.org/@types/console-log-level/-/console-log-level-1.4.0.tgz", + "integrity": "sha1-7/ccQa689RyLpa2LBdfVQkviuPM=" }, "@types/duplexify": { "version": "3.6.0", - "from": "@types/duplexify@>=3.5.0 <4.0.0", - "resolved": "https://registry.npmjs.org/@types/duplexify/-/duplexify-3.6.0.tgz" + "resolved": "https://registry.npmjs.org/@types/duplexify/-/duplexify-3.6.0.tgz", + "integrity": "sha1-38grZL06IWj1vSZESvFlvwI33Ng=", + "requires": { + "@types/node": "*" + } }, "@types/form-data": { "version": "2.2.1", - "from": "@types/form-data@*", - "resolved": "https://registry.npmjs.org/@types/form-data/-/form-data-2.2.1.tgz" + "resolved": "https://registry.npmjs.org/@types/form-data/-/form-data-2.2.1.tgz", + "integrity": "sha1-7is7jqoRwJOCiZU2BrdFtzjFSx4=", + "requires": { + "@types/node": "*" + } }, "@types/long": { "version": "4.0.0", - "from": "@types/long@>=4.0.0 <5.0.0", - "resolved": "https://registry.npmjs.org/@types/long/-/long-4.0.0.tgz" + "resolved": "https://registry.npmjs.org/@types/long/-/long-4.0.0.tgz", + "integrity": "sha1-cZVR0jUtMBrIuB23Mqy2vcKNve8=" }, "@types/node": { "version": "12.0.8", - "from": "@types/node@*", - "resolved": "https://registry.npmjs.org/@types/node/-/node-12.0.8.tgz" + "resolved": "https://registry.npmjs.org/@types/node/-/node-12.0.8.tgz", + "integrity": "sha1-VRRmvhGyrcPz1HFWdY9hC9n2sdg=" }, "@types/request": { "version": "2.48.1", - "from": "@types/request@>=2.47.0 <3.0.0", - "resolved": "https://registry.npmjs.org/@types/request/-/request-2.48.1.tgz" + "resolved": "https://registry.npmjs.org/@types/request/-/request-2.48.1.tgz", + "integrity": "sha1-5ALWkapmcPu/8ZV7FfEnAjCrQvo=", + "requires": { + "@types/caseless": "*", + "@types/form-data": "*", + "@types/node": "*", + "@types/tough-cookie": "*" + } }, "@types/semver": { "version": "5.5.0", - "from": "@types/semver@>=5.5.0 <6.0.0", - "resolved": "https://registry.npmjs.org/@types/semver/-/semver-5.5.0.tgz" + "resolved": "https://registry.npmjs.org/@types/semver/-/semver-5.5.0.tgz", + "integrity": "sha1-FGwqKe59O65L8vyydGNuJkyBPEU=" }, "@types/tough-cookie": { "version": "2.3.5", - "from": "@types/tough-cookie@*", - "resolved": "https://registry.npmjs.org/@types/tough-cookie/-/tough-cookie-2.3.5.tgz" + "resolved": "https://registry.npmjs.org/@types/tough-cookie/-/tough-cookie-2.3.5.tgz", + "integrity": "sha1-naRO11VxmZtlw3tgybK4jbVMWF0=" }, "abort-controller": { "version": "3.0.0", - "from": "abort-controller@>=3.0.0 <4.0.0", - "resolved": "https://registry.npmjs.org/abort-controller/-/abort-controller-3.0.0.tgz" + "resolved": "https://registry.npmjs.org/abort-controller/-/abort-controller-3.0.0.tgz", + "integrity": "sha1-6vVNU7YrrkE46AnKIlyEOabvs5I=", + "requires": { + "event-target-shim": "^5.0.0" + } }, "accepts": { "version": "1.3.5", - "from": "accepts@>=1.3.5 <1.4.0", - "resolved": "https://registry.npmjs.org/accepts/-/accepts-1.3.5.tgz" + "resolved": "https://registry.npmjs.org/accepts/-/accepts-1.3.5.tgz", + "integrity": "sha1-63d99gEXI6OxTopywIBcjoZ0a9I=", + "requires": { + "mime-types": "~2.1.18", + "negotiator": "0.6.1" + } }, "acorn": { "version": "6.1.1", - "from": "acorn@>=6.0.0 <7.0.0", - "resolved": "https://registry.npmjs.org/acorn/-/acorn-6.1.1.tgz" + "resolved": "https://registry.npmjs.org/acorn/-/acorn-6.1.1.tgz", + "integrity": "sha1-fSWuBbuK0fm2mRCOEJTs14hK3B8=" }, "active-x-obfuscator": { "version": "0.0.1", - "from": "active-x-obfuscator@0.0.1", - "resolved": "https://registry.npmjs.org/active-x-obfuscator/-/active-x-obfuscator-0.0.1.tgz" + "resolved": "https://registry.npmjs.org/active-x-obfuscator/-/active-x-obfuscator-0.0.1.tgz", + "integrity": "sha1-CJuJs3FF/x2ex0r2UwvlUmyuHxo=", + "requires": { + "zeparser": "0.0.5" + } }, "agent-base": { "version": "4.3.0", - "from": "agent-base@>=4.1.0 <5.0.0", - "resolved": "https://registry.npmjs.org/agent-base/-/agent-base-4.3.0.tgz" + "resolved": "https://registry.npmjs.org/agent-base/-/agent-base-4.3.0.tgz", + "integrity": "sha1-gWXwHENgCbzK0LHRIvBe13Dvxu4=", + "requires": { + "es6-promisify": "^5.0.0" + } }, "ajv": { "version": "6.7.0", - "from": "ajv@>=6.5.5 <7.0.0", - "resolved": "https://registry.npmjs.org/ajv/-/ajv-6.7.0.tgz" + "resolved": "https://registry.npmjs.org/ajv/-/ajv-6.7.0.tgz", + "integrity": "sha1-4857s3LWV3uxg58d/fy/WtKUjZY=", + "requires": { + "fast-deep-equal": "^2.0.1", + "fast-json-stable-stringify": "^2.0.0", + "json-schema-traverse": "^0.4.1", + "uri-js": "^4.2.2" + } }, "array-flatten": { "version": "1.1.1", - "from": "array-flatten@1.1.1", - "resolved": "https://registry.npmjs.org/array-flatten/-/array-flatten-1.1.1.tgz" + "resolved": "https://registry.npmjs.org/array-flatten/-/array-flatten-1.1.1.tgz", + "integrity": "sha1-ml9pkFGx5wczKPKgCJaLZOopVdI=" }, "arrify": { "version": "2.0.1", - "from": "arrify@>=2.0.0 <3.0.0", - "resolved": "https://registry.npmjs.org/arrify/-/arrify-2.0.1.tgz" + "resolved": "https://registry.npmjs.org/arrify/-/arrify-2.0.1.tgz", + "integrity": "sha1-yWVekzHgq81YjSp8rX6ZVvZnAfo=" }, "assert-plus": { "version": "1.0.0", - "from": "assert-plus@>=1.0.0 <2.0.0", - "resolved": "https://registry.npmjs.org/assert-plus/-/assert-plus-1.0.0.tgz" + "resolved": "https://registry.npmjs.org/assert-plus/-/assert-plus-1.0.0.tgz", + "integrity": "sha1-8S4PPF13sLHN2RRpQuTpbB5N1SU=" }, "assertion-error": { "version": "1.0.0", - "from": "assertion-error@1.0.0", "resolved": "https://registry.npmjs.org/assertion-error/-/assertion-error-1.0.0.tgz", + "integrity": "sha1-x/hUOP3UZrx8oWq5DIFRN5el0js=", "dev": true }, "async": { "version": "0.9.2", - "from": "async@>=0.9.0 <0.10.0", - "resolved": "https://registry.npmjs.org/async/-/async-0.9.2.tgz" + "resolved": "https://registry.npmjs.org/async/-/async-0.9.2.tgz", + "integrity": "sha1-rqdNXmHB+JlhO/ZL2mbUx48v0X0=" }, "async-listener": { "version": "0.6.10", - "from": "async-listener@>=0.6.0 <0.7.0", "resolved": "https://registry.npmjs.org/async-listener/-/async-listener-0.6.10.tgz", + "integrity": "sha1-p8l6vlcLpgLXgic8DeYKUePhfLw=", + "requires": { + "semver": "^5.3.0", + "shimmer": "^1.1.0" + }, "dependencies": { "semver": { "version": "5.7.0", - "from": "semver@>=5.3.0 <6.0.0", - "resolved": "https://registry.npmjs.org/semver/-/semver-5.7.0.tgz" + "resolved": "https://registry.npmjs.org/semver/-/semver-5.7.0.tgz", + "integrity": "sha1-eQp89v6lRZuslhELKbYEEtyP+Ws=" } } }, "asynckit": { "version": "0.4.0", - "from": "asynckit@>=0.4.0 <0.5.0", - "resolved": "https://registry.npmjs.org/asynckit/-/asynckit-0.4.0.tgz" + "resolved": "https://registry.npmjs.org/asynckit/-/asynckit-0.4.0.tgz", + "integrity": "sha1-x57Zf380y48robyXkLzDZkdLS3k=" }, "aws-sign2": { "version": "0.7.0", - "from": "aws-sign2@>=0.7.0 <0.8.0", - "resolved": "https://registry.npmjs.org/aws-sign2/-/aws-sign2-0.7.0.tgz" + "resolved": "https://registry.npmjs.org/aws-sign2/-/aws-sign2-0.7.0.tgz", + "integrity": "sha1-tG6JCTSpWR8tL2+G1+ap8bP+dqg=" }, "aws4": { "version": "1.8.0", - "from": "aws4@>=1.8.0 <2.0.0", - "resolved": "https://registry.npmjs.org/aws4/-/aws4-1.8.0.tgz" + "resolved": "https://registry.npmjs.org/aws4/-/aws4-1.8.0.tgz", + "integrity": "sha1-8OAD2cqef1nHpQiUXXsu+aBKVC8=" }, "axios": { "version": "0.18.1", - "from": "axios@>=0.18.0 <0.19.0", - "resolved": "https://registry.npmjs.org/axios/-/axios-0.18.1.tgz" + "resolved": "https://registry.npmjs.org/axios/-/axios-0.18.1.tgz", + "integrity": "sha1-/z8N4ue10YDnV62YAA8Qgbh7zqM=", + "requires": { + "follow-redirects": "1.5.10", + "is-buffer": "^2.0.2" + } }, "balanced-match": { "version": "1.0.0", - "from": "balanced-match@^1.0.0", - "resolved": "https://registry.npmjs.org/balanced-match/-/balanced-match-1.0.0.tgz" + "resolved": "https://registry.npmjs.org/balanced-match/-/balanced-match-1.0.0.tgz", + "integrity": "sha1-ibTRmasr7kneFk6gK4nORi1xt2c=" }, "base64-js": { "version": "1.3.0", - "from": "base64-js@>=1.3.0 <2.0.0", - "resolved": "https://registry.npmjs.org/base64-js/-/base64-js-1.3.0.tgz" + "resolved": "https://registry.npmjs.org/base64-js/-/base64-js-1.3.0.tgz", + "integrity": "sha1-yrHmEY8FEJXli1KBrqjBzSK/wOM=" }, "base64-url": { "version": "1.2.1", - "from": "base64-url@1.2.1", "resolved": "https://registry.npmjs.org/base64-url/-/base64-url-1.2.1.tgz", + "integrity": "sha1-GZ/WYXAqDnt9yubgaYuwicUvbXg=", "dev": true }, "base64id": { "version": "0.1.0", - "from": "base64id@0.1.0", - "resolved": "https://registry.npmjs.org/base64id/-/base64id-0.1.0.tgz" + "resolved": "https://registry.npmjs.org/base64id/-/base64id-0.1.0.tgz", + "integrity": "sha1-As4P3u4M709ACA4ec+g08LG/zj8=" }, "basic-auth-connect": { "version": "1.0.0", - "from": "basic-auth-connect@>=1.0.0 <2.0.0", - "resolved": "https://registry.npmjs.org/basic-auth-connect/-/basic-auth-connect-1.0.0.tgz" + "resolved": "https://registry.npmjs.org/basic-auth-connect/-/basic-auth-connect-1.0.0.tgz", + "integrity": "sha1-/bC0OWLKe0BFanwrtI/hc9otISI=" }, "bcrypt-pbkdf": { "version": "1.0.2", - "from": "bcrypt-pbkdf@>=1.0.0 <2.0.0", - "resolved": "https://registry.npmjs.org/bcrypt-pbkdf/-/bcrypt-pbkdf-1.0.2.tgz" + "resolved": "https://registry.npmjs.org/bcrypt-pbkdf/-/bcrypt-pbkdf-1.0.2.tgz", + "integrity": "sha1-pDAdOJtqQ/m2f/PKEaP2Y342Dp4=", + "requires": { + "tweetnacl": "^0.14.3" + } }, "bignumber.js": { "version": "7.2.1", - "from": "bignumber.js@>=7.0.0 <8.0.0", - "resolved": "https://registry.npmjs.org/bignumber.js/-/bignumber.js-7.2.1.tgz" + "resolved": "https://registry.npmjs.org/bignumber.js/-/bignumber.js-7.2.1.tgz", + "integrity": "sha1-gMBIdZ2CaACAfEv9Uh5Q7bulel8=" }, "bindings": { "version": "1.5.0", - "from": "bindings@>=1.2.1 <2.0.0", - "resolved": "https://registry.npmjs.org/bindings/-/bindings-1.5.0.tgz" + "resolved": "https://registry.npmjs.org/bindings/-/bindings-1.5.0.tgz", + "integrity": "sha1-EDU8npRTNLwFEabZCzj7x8nFBN8=", + "requires": { + "file-uri-to-path": "1.0.0" + } }, "bintrees": { "version": "1.0.1", - "from": "bintrees@1.0.1", - "resolved": "https://registry.npmjs.org/bintrees/-/bintrees-1.0.1.tgz" + "resolved": "https://registry.npmjs.org/bintrees/-/bintrees-1.0.1.tgz", + "integrity": "sha1-DmVcm5wkNeqraL9AJyJtK1WjRSQ=" }, "body-parser": { "version": "1.18.3", - "from": "body-parser@>=1.12.0 <2.0.0", - "resolved": "https://registry.npmjs.org/body-parser/-/body-parser-1.18.3.tgz" + "resolved": "https://registry.npmjs.org/body-parser/-/body-parser-1.18.3.tgz", + "integrity": "sha1-WykhmP/dVTs6DyDe0FkrlWlVyLQ=", + "requires": { + "bytes": "3.0.0", + "content-type": "~1.0.4", + "debug": "2.6.9", + "depd": "~1.1.2", + "http-errors": "~1.6.3", + "iconv-lite": "0.4.23", + "on-finished": "~2.3.0", + "qs": "6.5.2", + "raw-body": "2.3.3", + "type-is": "~1.6.16" + } }, "brace-expansion": { "version": "1.1.11", - "from": "brace-expansion@^1.1.7", - "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-1.1.11.tgz" + "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-1.1.11.tgz", + "integrity": "sha1-PH/L9SnYcibz0vUrlm/1Jx60Qd0=", + "requires": { + "balanced-match": "^1.0.0", + "concat-map": "0.0.1" + } }, "browser-stdout": { "version": "1.3.0", - "from": "browser-stdout@1.3.0", "resolved": "https://registry.npmjs.org/browser-stdout/-/browser-stdout-1.3.0.tgz", + "integrity": "sha1-81HTKWnTL6XXpVZxVCY9korjvR8=", "dev": true }, "buffer-equal-constant-time": { "version": "1.0.1", - "from": "buffer-equal-constant-time@1.0.1", - "resolved": "https://registry.npmjs.org/buffer-equal-constant-time/-/buffer-equal-constant-time-1.0.1.tgz" + "resolved": "https://registry.npmjs.org/buffer-equal-constant-time/-/buffer-equal-constant-time-1.0.1.tgz", + "integrity": "sha1-+OcRMvf/5uAaXJaXpMbz5I1cyBk=" }, "builtin-modules": { "version": "3.1.0", - "from": "builtin-modules@>=3.0.0 <4.0.0", - "resolved": "https://registry.npmjs.org/builtin-modules/-/builtin-modules-3.1.0.tgz" + "resolved": "https://registry.npmjs.org/builtin-modules/-/builtin-modules-3.1.0.tgz", + "integrity": "sha1-qtl8FRMet2tltQ7yCOdYTNdqdIQ=" }, "bunyan": { "version": "0.22.3", - "from": "bunyan@>=0.22.3 <0.23.0", "resolved": "https://registry.npmjs.org/bunyan/-/bunyan-0.22.3.tgz", - "dev": true + "integrity": "sha1-ehncG0yMZF90AkGnQPIkUUfGfsI=", + "dev": true, + "requires": { + "dtrace-provider": "0.2.8", + "mv": "~2" + } }, "bytes": { "version": "3.0.0", - "from": "bytes@3.0.0", - "resolved": "https://registry.npmjs.org/bytes/-/bytes-3.0.0.tgz" + "resolved": "https://registry.npmjs.org/bytes/-/bytes-3.0.0.tgz", + "integrity": "sha1-0ygVQE1olpn4Wk6k+odV3ROpYEg=" }, "caseless": { "version": "0.12.0", - "from": "caseless@>=0.12.0 <0.13.0", - "resolved": "https://registry.npmjs.org/caseless/-/caseless-0.12.0.tgz" + "resolved": "https://registry.npmjs.org/caseless/-/caseless-0.12.0.tgz", + "integrity": "sha1-G2gcIf+EAzyCZUMJBolCDRhxUdw=" }, "chai": { "version": "1.9.2", - "from": "chai@>=1.9.1 <1.10.0", "resolved": "https://registry.npmjs.org/chai/-/chai-1.9.2.tgz", - "dev": true + "integrity": "sha1-Pxog+CsLnXQ3V30k1vErGmnTtZA=", + "dev": true, + "requires": { + "assertion-error": "1.0.0", + "deep-eql": "0.1.3" + } }, "cluster-key-slot": { "version": "1.1.0", - "from": "cluster-key-slot@>=1.0.6 <2.0.0", - "resolved": "https://registry.npmjs.org/cluster-key-slot/-/cluster-key-slot-1.1.0.tgz" + "resolved": "https://registry.npmjs.org/cluster-key-slot/-/cluster-key-slot-1.1.0.tgz", + "integrity": "sha1-MEdLKpgfsSFyaVgzBSvA0BM20Q0=" }, "coffee-script": { "version": "1.6.0", - "from": "coffee-script@1.6.0", - "resolved": "https://registry.npmjs.org/coffee-script/-/coffee-script-1.6.0.tgz" + "resolved": "https://registry.npmjs.org/coffee-script/-/coffee-script-1.6.0.tgz", + "integrity": "sha1-gIs5bhEPU9AhoZpO8fZb4OjjX6M=" }, "combined-stream": { "version": "1.0.7", - "from": "combined-stream@>=1.0.6 <1.1.0", - "resolved": "https://registry.npmjs.org/combined-stream/-/combined-stream-1.0.7.tgz" + "resolved": "https://registry.npmjs.org/combined-stream/-/combined-stream-1.0.7.tgz", + "integrity": "sha1-LR0kMXr7ir6V1tLAsHtXgTU52Cg=", + "requires": { + "delayed-stream": "~1.0.0" + } }, "concat-map": { "version": "0.0.1", - "from": "concat-map@0.0.1", - "resolved": "https://registry.npmjs.org/concat-map/-/concat-map-0.0.1.tgz" + "resolved": "https://registry.npmjs.org/concat-map/-/concat-map-0.0.1.tgz", + "integrity": "sha1-2Klr13/Wjfd5OnMDajug1UBdR3s=" }, "connect-redis": { "version": "2.5.1", - "from": "connect-redis@>=2.1.0 <3.0.0", "resolved": "https://registry.npmjs.org/connect-redis/-/connect-redis-2.5.1.tgz", + "integrity": "sha1-6MCF227Gg7T8RXzzP5PD5+1vA/c=", + "requires": { + "debug": "^1.0.4", + "redis": "^0.12.1" + }, "dependencies": { "debug": { "version": "1.0.5", - "from": "debug@>=1.0.4 <2.0.0", - "resolved": "https://registry.npmjs.org/debug/-/debug-1.0.5.tgz" + "resolved": "https://registry.npmjs.org/debug/-/debug-1.0.5.tgz", + "integrity": "sha1-9yQSF0MPmd7EwrRz6rkiKOh0wqw=", + "requires": { + "ms": "2.0.0" + } } } }, "console-log-level": { "version": "1.4.1", - "from": "console-log-level@>=1.4.0 <2.0.0", - "resolved": "https://registry.npmjs.org/console-log-level/-/console-log-level-1.4.1.tgz" + "resolved": "https://registry.npmjs.org/console-log-level/-/console-log-level-1.4.1.tgz", + "integrity": "sha1-nFprue8e9lsFq6gwKLD/iUzfYwo=" }, "content-disposition": { "version": "0.5.2", - "from": "content-disposition@0.5.2", - "resolved": "https://registry.npmjs.org/content-disposition/-/content-disposition-0.5.2.tgz" + "resolved": "https://registry.npmjs.org/content-disposition/-/content-disposition-0.5.2.tgz", + "integrity": "sha1-DPaLud318r55YcOoUXjLhdunjLQ=" }, "content-type": { "version": "1.0.4", - "from": "content-type@>=1.0.4 <1.1.0", - "resolved": "https://registry.npmjs.org/content-type/-/content-type-1.0.4.tgz" + "resolved": "https://registry.npmjs.org/content-type/-/content-type-1.0.4.tgz", + "integrity": "sha1-4TjMdeBAxyexlm/l5fjJruJW/js=" }, "continuation-local-storage": { "version": "3.2.1", - "from": "continuation-local-storage@>=3.2.1 <4.0.0", - "resolved": "https://registry.npmjs.org/continuation-local-storage/-/continuation-local-storage-3.2.1.tgz" + "resolved": "https://registry.npmjs.org/continuation-local-storage/-/continuation-local-storage-3.2.1.tgz", + "integrity": "sha1-EfYT906RT+mzTJKtLSj+auHbf/s=", + "requires": { + "async-listener": "^0.6.0", + "emitter-listener": "^1.1.1" + } }, "cookie": { "version": "0.3.1", - "from": "cookie@0.3.1", - "resolved": "https://registry.npmjs.org/cookie/-/cookie-0.3.1.tgz" + "resolved": "https://registry.npmjs.org/cookie/-/cookie-0.3.1.tgz", + "integrity": "sha1-5+Ch+e9DtMi6klxcWpboBtFoc7s=" }, "cookie-parser": { "version": "1.4.3", - "from": "cookie-parser@>=1.3.3 <2.0.0", - "resolved": "https://registry.npmjs.org/cookie-parser/-/cookie-parser-1.4.3.tgz" + "resolved": "https://registry.npmjs.org/cookie-parser/-/cookie-parser-1.4.3.tgz", + "integrity": "sha1-D+MfoZ0AC5X0qt8fU/3CuKIDuqU=", + "requires": { + "cookie": "0.3.1", + "cookie-signature": "1.0.6" + } }, "cookie-signature": { "version": "1.0.6", - "from": "cookie-signature@1.0.6", - "resolved": "https://registry.npmjs.org/cookie-signature/-/cookie-signature-1.0.6.tgz" + "resolved": "https://registry.npmjs.org/cookie-signature/-/cookie-signature-1.0.6.tgz", + "integrity": "sha1-4wOogrNCzD7oylE6eZmXNNqzriw=" }, "core-util-is": { "version": "1.0.2", - "from": "core-util-is@>=1.0.0 <1.1.0", - "resolved": "https://registry.npmjs.org/core-util-is/-/core-util-is-1.0.2.tgz" + "resolved": "https://registry.npmjs.org/core-util-is/-/core-util-is-1.0.2.tgz", + "integrity": "sha1-tf1UIgqivFq1eqtxQMlAdUUDwac=" }, "crc": { "version": "3.4.4", - "from": "crc@3.4.4", - "resolved": "https://registry.npmjs.org/crc/-/crc-3.4.4.tgz" + "resolved": "https://registry.npmjs.org/crc/-/crc-3.4.4.tgz", + "integrity": "sha1-naHpgOO9RPxck79as9ozeNheRms=" }, "dashdash": { "version": "1.14.1", - "from": "dashdash@>=1.12.0 <2.0.0", "resolved": "https://registry.npmjs.org/dashdash/-/dashdash-1.14.1.tgz", + "integrity": "sha1-hTz6D3y+L+1d4gMmuN1YEDX24vA=", + "requires": { + "assert-plus": "^1.0.0" + }, "dependencies": { "assert-plus": { "version": "1.0.0", - "from": "assert-plus@^1.0.0", - "resolved": "https://registry.npmjs.org/assert-plus/-/assert-plus-1.0.0.tgz" + "resolved": "https://registry.npmjs.org/assert-plus/-/assert-plus-1.0.0.tgz", + "integrity": "sha1-8S4PPF13sLHN2RRpQuTpbB5N1SU=" } } }, "debug": { "version": "2.6.9", - "from": "debug@2.6.9", - "resolved": "https://registry.npmjs.org/debug/-/debug-2.6.9.tgz" + "resolved": "https://registry.npmjs.org/debug/-/debug-2.6.9.tgz", + "integrity": "sha1-XRKFFd8TT/Mn6QpMk/Tgd6U2NB8=", + "requires": { + "ms": "2.0.0" + } }, "deep-eql": { "version": "0.1.3", - "from": "deep-eql@0.1.3", "resolved": "https://registry.npmjs.org/deep-eql/-/deep-eql-0.1.3.tgz", - "dev": true + "integrity": "sha1-71WKyrjeJSBs1xOQbXTlaTDrafI=", + "dev": true, + "requires": { + "type-detect": "0.1.1" + } }, "define-properties": { "version": "1.1.3", - "from": "define-properties@>=1.1.3 <2.0.0", "resolved": "https://registry.npmjs.org/define-properties/-/define-properties-1.1.3.tgz", - "dev": true + "integrity": "sha1-z4jabL7ib+bbcJT2HYcMvYTO6fE=", + "dev": true, + "requires": { + "object-keys": "^1.0.12" + } }, "delay": { "version": "4.3.0", - "from": "delay@>=4.0.1 <5.0.0", - "resolved": "https://registry.npmjs.org/delay/-/delay-4.3.0.tgz" + "resolved": "https://registry.npmjs.org/delay/-/delay-4.3.0.tgz", + "integrity": "sha1-7+6/uPVFV5yzlrOnIkQ+yW0UxQ4=" }, "delayed-stream": { "version": "1.0.0", - "from": "delayed-stream@>=1.0.0 <1.1.0", - "resolved": "https://registry.npmjs.org/delayed-stream/-/delayed-stream-1.0.0.tgz" + "resolved": "https://registry.npmjs.org/delayed-stream/-/delayed-stream-1.0.0.tgz", + "integrity": "sha1-3zrhmayt+31ECqrgsp4icrJOxhk=" }, "denque": { "version": "1.4.1", - "from": "denque@>=1.1.0 <2.0.0", - "resolved": "https://registry.npmjs.org/denque/-/denque-1.4.1.tgz" + "resolved": "https://registry.npmjs.org/denque/-/denque-1.4.1.tgz", + "integrity": "sha1-Z0T/dkHBSMP4ppwwflEjXB9KN88=" }, "depd": { "version": "1.1.2", - "from": "depd@>=1.1.2 <1.2.0", - "resolved": "https://registry.npmjs.org/depd/-/depd-1.1.2.tgz" + "resolved": "https://registry.npmjs.org/depd/-/depd-1.1.2.tgz", + "integrity": "sha1-m81S4UwJd2PnSbJ0xDRu0uVgtak=" }, "destroy": { "version": "1.0.4", - "from": "destroy@>=1.0.4 <1.1.0", - "resolved": "https://registry.npmjs.org/destroy/-/destroy-1.0.4.tgz" + "resolved": "https://registry.npmjs.org/destroy/-/destroy-1.0.4.tgz", + "integrity": "sha1-l4hXRCxEdJ5CBmE+N5RiBYJqvYA=" }, "dtrace-provider": { "version": "0.2.8", - "from": "dtrace-provider@0.2.8", "resolved": "https://registry.npmjs.org/dtrace-provider/-/dtrace-provider-0.2.8.tgz", + "integrity": "sha1-4kPxkhmqlfvw2PL/sH9b1k6U/iA=", "dev": true, "optional": true }, "duplexify": { "version": "3.7.1", - "from": "duplexify@>=3.6.0 <4.0.0", - "resolved": "https://registry.npmjs.org/duplexify/-/duplexify-3.7.1.tgz" + "resolved": "https://registry.npmjs.org/duplexify/-/duplexify-3.7.1.tgz", + "integrity": "sha1-Kk31MX9sz9kfhtb9JdjYoQO4gwk=", + "requires": { + "end-of-stream": "^1.0.0", + "inherits": "^2.0.1", + "readable-stream": "^2.0.0", + "stream-shift": "^1.0.0" + } }, "ecc-jsbn": { "version": "0.1.2", - "from": "ecc-jsbn@>=0.1.1 <0.2.0", - "resolved": "https://registry.npmjs.org/ecc-jsbn/-/ecc-jsbn-0.1.2.tgz" + "resolved": "https://registry.npmjs.org/ecc-jsbn/-/ecc-jsbn-0.1.2.tgz", + "integrity": "sha1-OoOpBOVDUyh4dMVkt1SThoSamMk=", + "requires": { + "jsbn": "~0.1.0", + "safer-buffer": "^2.1.0" + } }, "ecdsa-sig-formatter": { "version": "1.0.11", - "from": "ecdsa-sig-formatter@1.0.11", - "resolved": "https://registry.npmjs.org/ecdsa-sig-formatter/-/ecdsa-sig-formatter-1.0.11.tgz" + "resolved": "https://registry.npmjs.org/ecdsa-sig-formatter/-/ecdsa-sig-formatter-1.0.11.tgz", + "integrity": "sha1-rg8PothQRe8UqBfao86azQSJ5b8=", + "requires": { + "safe-buffer": "^5.0.1" + } }, "ee-first": { "version": "1.1.1", - "from": "ee-first@1.1.1", - "resolved": "https://registry.npmjs.org/ee-first/-/ee-first-1.1.1.tgz" + "resolved": "https://registry.npmjs.org/ee-first/-/ee-first-1.1.1.tgz", + "integrity": "sha1-WQxhFWsK4vTwJVcyoViyZrxWsh0=" }, "emitter-listener": { "version": "1.1.2", - "from": "emitter-listener@>=1.1.1 <2.0.0", - "resolved": "https://registry.npmjs.org/emitter-listener/-/emitter-listener-1.1.2.tgz" + "resolved": "https://registry.npmjs.org/emitter-listener/-/emitter-listener-1.1.2.tgz", + "integrity": "sha1-VrFA6PaZI3Wz18ssqxzHQy2WMug=", + "requires": { + "shimmer": "^1.2.0" + } }, "encodeurl": { "version": "1.0.2", - "from": "encodeurl@>=1.0.2 <1.1.0", - "resolved": "https://registry.npmjs.org/encodeurl/-/encodeurl-1.0.2.tgz" + "resolved": "https://registry.npmjs.org/encodeurl/-/encodeurl-1.0.2.tgz", + "integrity": "sha1-rT/0yG7C0CkyL1oCw6mmBslbP1k=" }, "end-of-stream": { "version": "1.4.1", - "from": "end-of-stream@>=1.0.0 <2.0.0", - "resolved": "https://registry.npmjs.org/end-of-stream/-/end-of-stream-1.4.1.tgz" + "resolved": "https://registry.npmjs.org/end-of-stream/-/end-of-stream-1.4.1.tgz", + "integrity": "sha1-7SljTRm6ukY7bOa4CjchPqtx7EM=", + "requires": { + "once": "^1.4.0" + } }, "ent": { "version": "2.2.0", - "from": "ent@>=2.2.0 <3.0.0", - "resolved": "https://registry.npmjs.org/ent/-/ent-2.2.0.tgz" + "resolved": "https://registry.npmjs.org/ent/-/ent-2.2.0.tgz", + "integrity": "sha1-6WQhkyWiHQX0RGai9obtbOX13R0=" }, "es-abstract": { "version": "1.13.0", - "from": "es-abstract@>=1.12.0 <2.0.0", "resolved": "https://registry.npmjs.org/es-abstract/-/es-abstract-1.13.0.tgz", - "dev": true + "integrity": "sha1-rIYUX91QmdjdSVWMy6Lq+biOJOk=", + "dev": true, + "requires": { + "es-to-primitive": "^1.2.0", + "function-bind": "^1.1.1", + "has": "^1.0.3", + "is-callable": "^1.1.4", + "is-regex": "^1.0.4", + "object-keys": "^1.0.12" + } }, "es-to-primitive": { "version": "1.2.0", - "from": "es-to-primitive@>=1.2.0 <2.0.0", "resolved": "https://registry.npmjs.org/es-to-primitive/-/es-to-primitive-1.2.0.tgz", - "dev": true + "integrity": "sha1-7fckeAM0VujdqO8J4ArZZQcH83c=", + "dev": true, + "requires": { + "is-callable": "^1.1.4", + "is-date-object": "^1.0.1", + "is-symbol": "^1.0.2" + } }, "es6-promise": { "version": "4.2.8", - "from": "es6-promise@>=4.0.3 <5.0.0", - "resolved": "https://registry.npmjs.org/es6-promise/-/es6-promise-4.2.8.tgz" + "resolved": "https://registry.npmjs.org/es6-promise/-/es6-promise-4.2.8.tgz", + "integrity": "sha1-TrIVlMlyvEBVPSduUQU5FD21Pgo=" }, "es6-promisify": { "version": "5.0.0", - "from": "es6-promisify@>=5.0.0 <6.0.0", - "resolved": "https://registry.npmjs.org/es6-promisify/-/es6-promisify-5.0.0.tgz" + "resolved": "https://registry.npmjs.org/es6-promisify/-/es6-promisify-5.0.0.tgz", + "integrity": "sha1-UQnWLz5W6pZ8S2NQWu8IKRyKUgM=", + "requires": { + "es6-promise": "^4.0.3" + } }, "escape-html": { "version": "1.0.3", - "from": "escape-html@>=1.0.3 <1.1.0", - "resolved": "https://registry.npmjs.org/escape-html/-/escape-html-1.0.3.tgz" + "resolved": "https://registry.npmjs.org/escape-html/-/escape-html-1.0.3.tgz", + "integrity": "sha1-Aljq5NPQwJdN4cFpGI7wBR0dGYg=" }, "escape-string-regexp": { "version": "1.0.5", - "from": "escape-string-regexp@>=1.0.0 <2.0.0", "resolved": "https://registry.npmjs.org/escape-string-regexp/-/escape-string-regexp-1.0.5.tgz", + "integrity": "sha1-G2HAViGQqN/2rjuyzwIAyhMLhtQ=", "dev": true }, "etag": { "version": "1.8.1", - "from": "etag@>=1.8.1 <1.9.0", - "resolved": "https://registry.npmjs.org/etag/-/etag-1.8.1.tgz" + "resolved": "https://registry.npmjs.org/etag/-/etag-1.8.1.tgz", + "integrity": "sha1-Qa4u62XvpiJorr/qg6x9eSmbCIc=" }, "event-target-shim": { "version": "5.0.1", - "from": "event-target-shim@>=5.0.0 <6.0.0", - "resolved": "https://registry.npmjs.org/event-target-shim/-/event-target-shim-5.0.1.tgz" + "resolved": "https://registry.npmjs.org/event-target-shim/-/event-target-shim-5.0.1.tgz", + "integrity": "sha1-XU0+vflYPWOlMzzi3rdICrKwV4k=" }, "express": { "version": "4.16.3", - "from": "express@>=4.10.1 <5.0.0", "resolved": "https://registry.npmjs.org/express/-/express-4.16.3.tgz", + "integrity": "sha1-avilAjUNsyRuzEvs9rWjTSL37VM=", + "requires": { + "accepts": "~1.3.5", + "array-flatten": "1.1.1", + "body-parser": "1.18.2", + "content-disposition": "0.5.2", + "content-type": "~1.0.4", + "cookie": "0.3.1", + "cookie-signature": "1.0.6", + "debug": "2.6.9", + "depd": "~1.1.2", + "encodeurl": "~1.0.2", + "escape-html": "~1.0.3", + "etag": "~1.8.1", + "finalhandler": "1.1.1", + "fresh": "0.5.2", + "merge-descriptors": "1.0.1", + "methods": "~1.1.2", + "on-finished": "~2.3.0", + "parseurl": "~1.3.2", + "path-to-regexp": "0.1.7", + "proxy-addr": "~2.0.3", + "qs": "6.5.1", + "range-parser": "~1.2.0", + "safe-buffer": "5.1.1", + "send": "0.16.2", + "serve-static": "1.13.2", + "setprototypeof": "1.1.0", + "statuses": "~1.4.0", + "type-is": "~1.6.16", + "utils-merge": "1.0.1", + "vary": "~1.1.2" + }, "dependencies": { "body-parser": { "version": "1.18.2", - "from": "body-parser@1.18.2", - "resolved": "https://registry.npmjs.org/body-parser/-/body-parser-1.18.2.tgz" + "resolved": "https://registry.npmjs.org/body-parser/-/body-parser-1.18.2.tgz", + "integrity": "sha1-h2eKGdhLR9hZuDGZvVm84iKxBFQ=", + "requires": { + "bytes": "3.0.0", + "content-type": "~1.0.4", + "debug": "2.6.9", + "depd": "~1.1.1", + "http-errors": "~1.6.2", + "iconv-lite": "0.4.19", + "on-finished": "~2.3.0", + "qs": "6.5.1", + "raw-body": "2.3.2", + "type-is": "~1.6.15" + } }, "iconv-lite": { "version": "0.4.19", - "from": "iconv-lite@0.4.19", - "resolved": "https://registry.npmjs.org/iconv-lite/-/iconv-lite-0.4.19.tgz" + "resolved": "https://registry.npmjs.org/iconv-lite/-/iconv-lite-0.4.19.tgz", + "integrity": "sha1-90aPYBNfXl2tM5nAqBvpoWA6CCs=" }, "qs": { "version": "6.5.1", - "from": "qs@6.5.1", - "resolved": "https://registry.npmjs.org/qs/-/qs-6.5.1.tgz" + "resolved": "https://registry.npmjs.org/qs/-/qs-6.5.1.tgz", + "integrity": "sha1-NJzfbu+J7EXBLX1es/wMhwNDptg=" }, "raw-body": { "version": "2.3.2", - "from": "raw-body@2.3.2", "resolved": "https://registry.npmjs.org/raw-body/-/raw-body-2.3.2.tgz", + "integrity": "sha1-vNYMd9Prk83gBQKVw/N5OJvIj4k=", + "requires": { + "bytes": "3.0.0", + "http-errors": "1.6.2", + "iconv-lite": "0.4.19", + "unpipe": "1.0.0" + }, "dependencies": { "depd": { "version": "1.1.1", - "from": "depd@1.1.1", - "resolved": "https://registry.npmjs.org/depd/-/depd-1.1.1.tgz" + "resolved": "https://registry.npmjs.org/depd/-/depd-1.1.1.tgz", + "integrity": "sha1-V4O04cRZ8G+lyif5kfPQbnoxA1k=" }, "http-errors": { "version": "1.6.2", - "from": "http-errors@1.6.2", - "resolved": "https://registry.npmjs.org/http-errors/-/http-errors-1.6.2.tgz" + "resolved": "https://registry.npmjs.org/http-errors/-/http-errors-1.6.2.tgz", + "integrity": "sha1-CgAsyFcHGSp+eUbO7cERVfYOxzY=", + "requires": { + "depd": "1.1.1", + "inherits": "2.0.3", + "setprototypeof": "1.0.3", + "statuses": ">= 1.3.1 < 2" + } }, "setprototypeof": { "version": "1.0.3", - "from": "setprototypeof@1.0.3", - "resolved": "https://registry.npmjs.org/setprototypeof/-/setprototypeof-1.0.3.tgz" + "resolved": "https://registry.npmjs.org/setprototypeof/-/setprototypeof-1.0.3.tgz", + "integrity": "sha1-ZlZ+NwQ+608E2RvWWMDL77VbjgQ=" } } }, "statuses": { "version": "1.4.0", - "from": "statuses@>=1.4.0 <1.5.0", - "resolved": "https://registry.npmjs.org/statuses/-/statuses-1.4.0.tgz" + "resolved": "https://registry.npmjs.org/statuses/-/statuses-1.4.0.tgz", + "integrity": "sha1-u3PURtonlhBu/MG2AaJT1sRr0Ic=" } } }, "express-session": { "version": "1.15.6", - "from": "express-session@>=1.9.1 <2.0.0", "resolved": "https://registry.npmjs.org/express-session/-/express-session-1.15.6.tgz", + "integrity": "sha1-R7QWDIj0KrcP6KUI4xy/92dXqwo=", + "requires": { + "cookie": "0.3.1", + "cookie-signature": "1.0.6", + "crc": "3.4.4", + "debug": "2.6.9", + "depd": "~1.1.1", + "on-headers": "~1.0.1", + "parseurl": "~1.3.2", + "uid-safe": "~2.1.5", + "utils-merge": "1.0.1" + }, "dependencies": { "uid-safe": { "version": "2.1.5", - "from": "uid-safe@>=2.1.5 <2.2.0", - "resolved": "https://registry.npmjs.org/uid-safe/-/uid-safe-2.1.5.tgz" + "resolved": "https://registry.npmjs.org/uid-safe/-/uid-safe-2.1.5.tgz", + "integrity": "sha1-Kz1cckDo/C5Y+Komnl7knAhXvTo=", + "requires": { + "random-bytes": "~1.0.0" + } } } }, "extend": { "version": "3.0.2", - "from": "extend@>=3.0.1 <4.0.0", - "resolved": "https://registry.npmjs.org/extend/-/extend-3.0.2.tgz" + "resolved": "https://registry.npmjs.org/extend/-/extend-3.0.2.tgz", + "integrity": "sha1-+LETa0Bx+9jrFAr/hYsQGewpFfo=" }, "extsprintf": { "version": "1.3.0", - "from": "extsprintf@1.3.0", - "resolved": "https://registry.npmjs.org/extsprintf/-/extsprintf-1.3.0.tgz" + "resolved": "https://registry.npmjs.org/extsprintf/-/extsprintf-1.3.0.tgz", + "integrity": "sha1-lpGEQOMEGnpBT4xS48V06zw+HgU=" }, "fast-deep-equal": { "version": "2.0.1", - "from": "fast-deep-equal@>=2.0.1 <3.0.0", - "resolved": "https://registry.npmjs.org/fast-deep-equal/-/fast-deep-equal-2.0.1.tgz" + "resolved": "https://registry.npmjs.org/fast-deep-equal/-/fast-deep-equal-2.0.1.tgz", + "integrity": "sha1-ewUhjd+WZ79/Nwv3/bLLFf3Qqkk=" }, "fast-json-stable-stringify": { "version": "2.0.0", - "from": "fast-json-stable-stringify@>=2.0.0 <3.0.0", - "resolved": "https://registry.npmjs.org/fast-json-stable-stringify/-/fast-json-stable-stringify-2.0.0.tgz" + "resolved": "https://registry.npmjs.org/fast-json-stable-stringify/-/fast-json-stable-stringify-2.0.0.tgz", + "integrity": "sha1-1RQsDK7msRifh9OnYREGT4bIu/I=" }, "fast-text-encoding": { "version": "1.0.0", - "from": "fast-text-encoding@>=1.0.0 <2.0.0", - "resolved": "https://registry.npmjs.org/fast-text-encoding/-/fast-text-encoding-1.0.0.tgz" + "resolved": "https://registry.npmjs.org/fast-text-encoding/-/fast-text-encoding-1.0.0.tgz", + "integrity": "sha1-PlzoKTQJz6pxd6cbnKhOGx5vJe8=" }, "file-uri-to-path": { "version": "1.0.0", - "from": "file-uri-to-path@1.0.0", - "resolved": "https://registry.npmjs.org/file-uri-to-path/-/file-uri-to-path-1.0.0.tgz" + "resolved": "https://registry.npmjs.org/file-uri-to-path/-/file-uri-to-path-1.0.0.tgz", + "integrity": "sha1-VTp7hEb/b2hDWcRF8eN6BdrMM90=" }, "finalhandler": { "version": "1.1.1", - "from": "finalhandler@1.1.1", "resolved": "https://registry.npmjs.org/finalhandler/-/finalhandler-1.1.1.tgz", + "integrity": "sha512-Y1GUDo39ez4aHAw7MysnUD5JzYX+WaIj8I57kO3aEPT1fFRL4sr7mjei97FgnwhAyyzRYmQZaTHb2+9uZ1dPtg==", + "requires": { + "debug": "2.6.9", + "encodeurl": "~1.0.2", + "escape-html": "~1.0.3", + "on-finished": "~2.3.0", + "parseurl": "~1.3.2", + "statuses": "~1.4.0", + "unpipe": "~1.0.0" + }, "dependencies": { "statuses": { "version": "1.4.0", - "from": "statuses@>=1.4.0 <1.5.0", - "resolved": "https://registry.npmjs.org/statuses/-/statuses-1.4.0.tgz" + "resolved": "https://registry.npmjs.org/statuses/-/statuses-1.4.0.tgz", + "integrity": "sha512-zhSCtt8v2NDrRlPQpCNtw/heZLtfUDqxBM1udqikb/Hbk52LK4nQSwr10u77iopCW5LsyHpuXS0GnEc48mLeew==" } } }, "findit2": { "version": "2.2.3", - "from": "findit2@>=2.2.3 <3.0.0", - "resolved": "https://registry.npmjs.org/findit2/-/findit2-2.2.3.tgz" + "resolved": "https://registry.npmjs.org/findit2/-/findit2-2.2.3.tgz", + "integrity": "sha1-WKRmaX34piBc39vzlVNri9d3pfY=" }, "follow-redirects": { "version": "1.5.10", - "from": "follow-redirects@1.5.10", "resolved": "https://registry.npmjs.org/follow-redirects/-/follow-redirects-1.5.10.tgz", + "integrity": "sha1-e3qfmuov3/NnhqlP9kPtB/T/Xio=", + "requires": { + "debug": "=3.1.0" + }, "dependencies": { "debug": { "version": "3.1.0", - "from": "debug@3.1.0", - "resolved": "https://registry.npmjs.org/debug/-/debug-3.1.0.tgz" + "resolved": "https://registry.npmjs.org/debug/-/debug-3.1.0.tgz", + "integrity": "sha1-W7WgZyYotkFJVmuhaBnmFRjGcmE=", + "requires": { + "ms": "2.0.0" + } } } }, "forever-agent": { "version": "0.6.1", - "from": "forever-agent@>=0.6.1 <0.7.0", - "resolved": "https://registry.npmjs.org/forever-agent/-/forever-agent-0.6.1.tgz" + "resolved": "https://registry.npmjs.org/forever-agent/-/forever-agent-0.6.1.tgz", + "integrity": "sha1-+8cfDEGt6zf5bFd60e1C2P2sypE=" }, "form-data": { "version": "2.3.3", - "from": "form-data@>=2.3.2 <2.4.0", - "resolved": "https://registry.npmjs.org/form-data/-/form-data-2.3.3.tgz" + "resolved": "https://registry.npmjs.org/form-data/-/form-data-2.3.3.tgz", + "integrity": "sha1-3M5SwF9kTymManq5Nr1yTO/786Y=", + "requires": { + "asynckit": "^0.4.0", + "combined-stream": "^1.0.6", + "mime-types": "^2.1.12" + } }, "formatio": { "version": "1.1.1", - "from": "formatio@1.1.1", "resolved": "https://registry.npmjs.org/formatio/-/formatio-1.1.1.tgz", - "dev": true + "integrity": "sha1-XtPM1jZVEJc4NGXZlhmRAOhhYek=", + "dev": true, + "requires": { + "samsam": "~1.1" + } }, "forwarded": { "version": "0.1.2", - "from": "forwarded@>=0.1.2 <0.2.0", - "resolved": "https://registry.npmjs.org/forwarded/-/forwarded-0.1.2.tgz" + "resolved": "https://registry.npmjs.org/forwarded/-/forwarded-0.1.2.tgz", + "integrity": "sha1-mMI9qxF1ZXuMBXPozszZGw/xjIQ=" }, "fresh": { "version": "0.5.2", - "from": "fresh@0.5.2", - "resolved": "https://registry.npmjs.org/fresh/-/fresh-0.5.2.tgz" + "resolved": "https://registry.npmjs.org/fresh/-/fresh-0.5.2.tgz", + "integrity": "sha1-PYyt2Q2XZWn6g1qx+OSyOhBWBac=" }, "fs.realpath": { "version": "1.0.0", - "from": "fs.realpath@>=1.0.0 <2.0.0", "resolved": "https://registry.npmjs.org/fs.realpath/-/fs.realpath-1.0.0.tgz", + "integrity": "sha1-FQStJSMVjKpA20onh8sBQRmU6k8=", "dev": true }, "function-bind": { "version": "1.1.1", - "from": "function-bind@>=1.1.1 <2.0.0", "resolved": "https://registry.npmjs.org/function-bind/-/function-bind-1.1.1.tgz", + "integrity": "sha1-pWiZ0+o8m6uHS7l3O3xe3pL0iV0=", "dev": true }, "gaxios": { "version": "1.8.4", - "from": "gaxios@>=1.2.1 <2.0.0", - "resolved": "https://registry.npmjs.org/gaxios/-/gaxios-1.8.4.tgz" + "resolved": "https://registry.npmjs.org/gaxios/-/gaxios-1.8.4.tgz", + "integrity": "sha1-4Iw0/pPAqbZ6Ure556ZOZDX5ozk=", + "requires": { + "abort-controller": "^3.0.0", + "extend": "^3.0.2", + "https-proxy-agent": "^2.2.1", + "node-fetch": "^2.3.0" + } }, "gcp-metadata": { "version": "1.0.0", - "from": "gcp-metadata@>=1.0.0 <2.0.0", - "resolved": "https://registry.npmjs.org/gcp-metadata/-/gcp-metadata-1.0.0.tgz" + "resolved": "https://registry.npmjs.org/gcp-metadata/-/gcp-metadata-1.0.0.tgz", + "integrity": "sha1-UhJEAin6CZ/C98KlzcuVV16bLKY=", + "requires": { + "gaxios": "^1.0.2", + "json-bigint": "^0.3.0" + } }, "getpass": { "version": "0.1.7", - "from": "getpass@>=0.1.1 <0.2.0", "resolved": "https://registry.npmjs.org/getpass/-/getpass-0.1.7.tgz", + "integrity": "sha1-Xv+OPmhNVprkyysSgmBOi6YhSfo=", + "requires": { + "assert-plus": "^1.0.0" + }, "dependencies": { "assert-plus": { "version": "1.0.0", - "from": "assert-plus@^1.0.0", - "resolved": "https://registry.npmjs.org/assert-plus/-/assert-plus-1.0.0.tgz" + "resolved": "https://registry.npmjs.org/assert-plus/-/assert-plus-1.0.0.tgz", + "integrity": "sha1-8S4PPF13sLHN2RRpQuTpbB5N1SU=" } } }, "glob": { "version": "6.0.4", - "from": "glob@^6.0.1", "resolved": "https://registry.npmjs.org/glob/-/glob-6.0.4.tgz", - "optional": true + "integrity": "sha1-DwiGD2oVUSey+t1PnOJLGqtuTSI=", + "optional": true, + "requires": { + "inflight": "^1.0.4", + "inherits": "2", + "minimatch": "2 || 3", + "once": "^1.3.0", + "path-is-absolute": "^1.0.0" + } }, "google-auth-library": { "version": "3.1.2", - "from": "google-auth-library@>=3.1.1 <4.0.0", "resolved": "https://registry.npmjs.org/google-auth-library/-/google-auth-library-3.1.2.tgz", + "integrity": "sha1-/y+IzVzSEYpXvT1a08CTyIN/w1A=", + "requires": { + "base64-js": "^1.3.0", + "fast-text-encoding": "^1.0.0", + "gaxios": "^1.2.1", + "gcp-metadata": "^1.0.0", + "gtoken": "^2.3.2", + "https-proxy-agent": "^2.2.1", + "jws": "^3.1.5", + "lru-cache": "^5.0.0", + "semver": "^5.5.0" + }, "dependencies": { "semver": { "version": "5.7.0", - "from": "semver@>=5.5.0 <6.0.0", - "resolved": "https://registry.npmjs.org/semver/-/semver-5.7.0.tgz" + "resolved": "https://registry.npmjs.org/semver/-/semver-5.7.0.tgz", + "integrity": "sha1-eQp89v6lRZuslhELKbYEEtyP+Ws=" } } }, "google-p12-pem": { "version": "1.0.4", - "from": "google-p12-pem@>=1.0.0 <2.0.0", - "resolved": "https://registry.npmjs.org/google-p12-pem/-/google-p12-pem-1.0.4.tgz" + "resolved": "https://registry.npmjs.org/google-p12-pem/-/google-p12-pem-1.0.4.tgz", + "integrity": "sha1-t3+4M6Lrn388aJ4uVPCVJ293dgU=", + "requires": { + "node-forge": "^0.8.0", + "pify": "^4.0.0" + } }, "gtoken": { "version": "2.3.3", - "from": "gtoken@>=2.3.2 <3.0.0", "resolved": "https://registry.npmjs.org/gtoken/-/gtoken-2.3.3.tgz", + "integrity": "sha1-in/hVcXODEtxyIbPsoKpBg2UpkE=", + "requires": { + "gaxios": "^1.0.4", + "google-p12-pem": "^1.0.0", + "jws": "^3.1.5", + "mime": "^2.2.0", + "pify": "^4.0.0" + }, "dependencies": { "mime": { "version": "2.4.4", - "from": "mime@>=2.2.0 <3.0.0", - "resolved": "https://registry.npmjs.org/mime/-/mime-2.4.4.tgz" + "resolved": "https://registry.npmjs.org/mime/-/mime-2.4.4.tgz", + "integrity": "sha1-vXuRE1/GsBzePpuuM9ZZtj2IV+U=" } } }, "har-schema": { "version": "2.0.0", - "from": "har-schema@>=2.0.0 <3.0.0", - "resolved": "https://registry.npmjs.org/har-schema/-/har-schema-2.0.0.tgz" + "resolved": "https://registry.npmjs.org/har-schema/-/har-schema-2.0.0.tgz", + "integrity": "sha1-qUwiJOvKwEeCoNkDVSHyRzW37JI=" }, "har-validator": { "version": "5.1.3", - "from": "har-validator@>=5.1.0 <5.2.0", - "resolved": "https://registry.npmjs.org/har-validator/-/har-validator-5.1.3.tgz" + "resolved": "https://registry.npmjs.org/har-validator/-/har-validator-5.1.3.tgz", + "integrity": "sha1-HvievT5JllV2de7ZiTEQ3DUPoIA=", + "requires": { + "ajv": "^6.5.5", + "har-schema": "^2.0.0" + } }, "has": { "version": "1.0.3", - "from": "has@>=1.0.3 <2.0.0", "resolved": "https://registry.npmjs.org/has/-/has-1.0.3.tgz", - "dev": true + "integrity": "sha1-ci18v8H2qoJB8W3YFOAR4fQeh5Y=", + "dev": true, + "requires": { + "function-bind": "^1.1.1" + } }, "has-symbols": { "version": "1.0.0", - "from": "has-symbols@>=1.0.0 <2.0.0", "resolved": "https://registry.npmjs.org/has-symbols/-/has-symbols-1.0.0.tgz", + "integrity": "sha1-uhqPGvKg/DllD1yFA2dwQSIGO0Q=", "dev": true }, "he": { "version": "1.1.1", - "from": "he@1.1.1", "resolved": "https://registry.npmjs.org/he/-/he-1.1.1.tgz", + "integrity": "sha1-k0EP0hsAlzUVH4howvJx80J+I/0=", "dev": true }, "hex2dec": { "version": "1.1.2", - "from": "hex2dec@>=1.0.1 <2.0.0", - "resolved": "https://registry.npmjs.org/hex2dec/-/hex2dec-1.1.2.tgz" + "resolved": "https://registry.npmjs.org/hex2dec/-/hex2dec-1.1.2.tgz", + "integrity": "sha1-jhzkvvNqdPfVcjw/swkMKGAHczg=" }, "http-errors": { "version": "1.6.3", - "from": "http-errors@>=1.6.3 <1.7.0", - "resolved": "https://registry.npmjs.org/http-errors/-/http-errors-1.6.3.tgz" + "resolved": "https://registry.npmjs.org/http-errors/-/http-errors-1.6.3.tgz", + "integrity": "sha1-i1VoC7S+KDoLW/TqLjhYC+HZMg0=", + "requires": { + "depd": "~1.1.2", + "inherits": "2.0.3", + "setprototypeof": "1.1.0", + "statuses": ">= 1.4.0 < 2" + } }, "http-signature": { "version": "1.2.0", - "from": "http-signature@>=1.2.0 <1.3.0", - "resolved": "https://registry.npmjs.org/http-signature/-/http-signature-1.2.0.tgz" + "resolved": "https://registry.npmjs.org/http-signature/-/http-signature-1.2.0.tgz", + "integrity": "sha1-muzZJRFHcvPZW2WmCruPfBj7rOE=", + "requires": { + "assert-plus": "^1.0.0", + "jsprim": "^1.2.2", + "sshpk": "^1.7.0" + } }, "https-proxy-agent": { "version": "2.2.1", - "from": "https-proxy-agent@>=2.2.1 <3.0.0", "resolved": "https://registry.npmjs.org/https-proxy-agent/-/https-proxy-agent-2.2.1.tgz", + "integrity": "sha1-UVUpcPoE1yPgTFbQQXjD+SWSu8A=", + "requires": { + "agent-base": "^4.1.0", + "debug": "^3.1.0" + }, "dependencies": { "debug": { "version": "3.2.6", - "from": "debug@>=3.1.0 <4.0.0", - "resolved": "https://registry.npmjs.org/debug/-/debug-3.2.6.tgz" + "resolved": "https://registry.npmjs.org/debug/-/debug-3.2.6.tgz", + "integrity": "sha1-6D0X3hbYp++3cX7b5fsQE17uYps=", + "requires": { + "ms": "^2.1.1" + } }, "ms": { "version": "2.1.2", - "from": "ms@>=2.1.1 <3.0.0", - "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.2.tgz" + "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.2.tgz", + "integrity": "sha1-0J0fNXtEP0kzgqjrPM0YOHKuYAk=" } } }, "iconv-lite": { "version": "0.4.23", - "from": "iconv-lite@0.4.23", - "resolved": "https://registry.npmjs.org/iconv-lite/-/iconv-lite-0.4.23.tgz" + "resolved": "https://registry.npmjs.org/iconv-lite/-/iconv-lite-0.4.23.tgz", + "integrity": "sha1-KXhx9jvlB63Pv8pxXQzQ7thOmmM=", + "requires": { + "safer-buffer": ">= 2.1.2 < 3" + } }, "inflight": { "version": "1.0.6", - "from": "inflight@^1.0.4", - "resolved": "https://registry.npmjs.org/inflight/-/inflight-1.0.6.tgz" + "resolved": "https://registry.npmjs.org/inflight/-/inflight-1.0.6.tgz", + "integrity": "sha1-Sb1jMdfQLQwJvJEKEHW6gWW1bfk=", + "requires": { + "once": "^1.3.0", + "wrappy": "1" + } }, "inherits": { "version": "2.0.3", - "from": "inherits@2.0.3", - "resolved": "https://registry.npmjs.org/inherits/-/inherits-2.0.3.tgz" + "resolved": "https://registry.npmjs.org/inherits/-/inherits-2.0.3.tgz", + "integrity": "sha1-Yzwsg+PaQqUC9SRmAiSA9CCCYd4=" }, "ioredis": { "version": "4.14.1", - "from": "ioredis@>=4.14.1 <4.15.0", "resolved": "https://registry.npmjs.org/ioredis/-/ioredis-4.14.1.tgz", + "integrity": "sha1-tz3tlfzyIPEG0zElqS72ITqjExg=", + "requires": { + "cluster-key-slot": "^1.1.0", + "debug": "^4.1.1", + "denque": "^1.1.0", + "lodash.defaults": "^4.2.0", + "lodash.flatten": "^4.4.0", + "redis-commands": "1.5.0", + "redis-errors": "^1.2.0", + "redis-parser": "^3.0.0", + "standard-as-callback": "^2.0.1" + }, "dependencies": { "debug": { "version": "4.1.1", - "from": "debug@>=4.1.1 <5.0.0", - "resolved": "https://registry.npmjs.org/debug/-/debug-4.1.1.tgz" + "resolved": "https://registry.npmjs.org/debug/-/debug-4.1.1.tgz", + "integrity": "sha1-O3ImAlUQnGtYnO4FDx1RYTlmR5E=", + "requires": { + "ms": "^2.1.1" + } }, "ms": { "version": "2.1.2", - "from": "ms@>=2.1.1 <3.0.0", - "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.2.tgz" + "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.2.tgz", + "integrity": "sha1-0J0fNXtEP0kzgqjrPM0YOHKuYAk=" } } }, "ipaddr.js": { "version": "1.6.0", - "from": "ipaddr.js@1.6.0", - "resolved": "https://registry.npmjs.org/ipaddr.js/-/ipaddr.js-1.6.0.tgz" + "resolved": "https://registry.npmjs.org/ipaddr.js/-/ipaddr.js-1.6.0.tgz", + "integrity": "sha1-4/o1e3c9phnybpXwSdBVxyeW+Gs=" }, "is": { "version": "3.3.0", - "from": "is@>=3.2.0 <4.0.0", - "resolved": "https://registry.npmjs.org/is/-/is-3.3.0.tgz" + "resolved": "https://registry.npmjs.org/is/-/is-3.3.0.tgz", + "integrity": "sha1-Yc/23TxBk9uUo9YlggcrROVkXXk=" }, "is-arguments": { "version": "1.0.4", - "from": "is-arguments@>=1.0.4 <2.0.0", "resolved": "https://registry.npmjs.org/is-arguments/-/is-arguments-1.0.4.tgz", + "integrity": "sha1-P6+WbHy6D/Q3+zH2JQCC/PBEjPM=", "dev": true }, "is-buffer": { "version": "2.0.3", - "from": "is-buffer@>=2.0.2 <3.0.0", - "resolved": "https://registry.npmjs.org/is-buffer/-/is-buffer-2.0.3.tgz" + "resolved": "https://registry.npmjs.org/is-buffer/-/is-buffer-2.0.3.tgz", + "integrity": "sha1-Ts8/z3ScvR5HJonhCaxmJhol5yU=" }, "is-callable": { "version": "1.1.4", - "from": "is-callable@>=1.1.4 <2.0.0", "resolved": "https://registry.npmjs.org/is-callable/-/is-callable-1.1.4.tgz", + "integrity": "sha1-HhrfIZ4e62hNaR+dagX/DTCiTXU=", "dev": true }, "is-date-object": { "version": "1.0.1", - "from": "is-date-object@>=1.0.1 <2.0.0", "resolved": "https://registry.npmjs.org/is-date-object/-/is-date-object-1.0.1.tgz", + "integrity": "sha1-mqIOtq7rv/d/vTPnTKAbM1gdOhY=", "dev": true }, "is-generator-function": { "version": "1.0.7", - "from": "is-generator-function@>=1.0.7 <2.0.0", "resolved": "https://registry.npmjs.org/is-generator-function/-/is-generator-function-1.0.7.tgz", + "integrity": "sha1-0hMuUpuwAAp/gHlNS99c1eWBNSI=", "dev": true }, "is-regex": { "version": "1.0.4", - "from": "is-regex@>=1.0.4 <2.0.0", "resolved": "https://registry.npmjs.org/is-regex/-/is-regex-1.0.4.tgz", - "dev": true + "integrity": "sha1-VRdIm1RwkbCTDglWVM7SXul+lJE=", + "dev": true, + "requires": { + "has": "^1.0.1" + } }, "is-symbol": { "version": "1.0.2", - "from": "is-symbol@>=1.0.2 <2.0.0", "resolved": "https://registry.npmjs.org/is-symbol/-/is-symbol-1.0.2.tgz", - "dev": true + "integrity": "sha1-oFX2rlcZLK7jKeeoYBGLSXqVDzg=", + "dev": true, + "requires": { + "has-symbols": "^1.0.0" + } }, "is-typedarray": { "version": "1.0.0", - "from": "is-typedarray@>=1.0.0 <1.1.0", - "resolved": "https://registry.npmjs.org/is-typedarray/-/is-typedarray-1.0.0.tgz" + "resolved": "https://registry.npmjs.org/is-typedarray/-/is-typedarray-1.0.0.tgz", + "integrity": "sha1-5HnICFjfDBsR3dppQPlgEfzaSpo=" }, "isarray": { "version": "1.0.0", - "from": "isarray@>=1.0.0 <1.1.0", - "resolved": "https://registry.npmjs.org/isarray/-/isarray-1.0.0.tgz" + "resolved": "https://registry.npmjs.org/isarray/-/isarray-1.0.0.tgz", + "integrity": "sha1-u5NdSFgsuhaMBoNJV6VKPgcSTxE=" }, "isstream": { "version": "0.1.2", - "from": "isstream@>=0.1.2 <0.2.0", - "resolved": "https://registry.npmjs.org/isstream/-/isstream-0.1.2.tgz" + "resolved": "https://registry.npmjs.org/isstream/-/isstream-0.1.2.tgz", + "integrity": "sha1-R+Y/evVa+m+S4VAOaQ64uFKcCZo=" }, "jsbn": { "version": "0.1.1", - "from": "jsbn@>=0.1.0 <0.2.0", - "resolved": "https://registry.npmjs.org/jsbn/-/jsbn-0.1.1.tgz" + "resolved": "https://registry.npmjs.org/jsbn/-/jsbn-0.1.1.tgz", + "integrity": "sha1-peZUwuWi3rXyAdls77yoDA7y9RM=" }, "json-bigint": { "version": "0.3.0", - "from": "json-bigint@>=0.3.0 <0.4.0", - "resolved": "https://registry.npmjs.org/json-bigint/-/json-bigint-0.3.0.tgz" + "resolved": "https://registry.npmjs.org/json-bigint/-/json-bigint-0.3.0.tgz", + "integrity": "sha1-DM2RLEuCcNBfBW+9E4FLU9OCWx4=", + "requires": { + "bignumber.js": "^7.0.0" + } }, "json-schema": { "version": "0.2.3", - "from": "json-schema@0.2.3", - "resolved": "https://registry.npmjs.org/json-schema/-/json-schema-0.2.3.tgz" + "resolved": "https://registry.npmjs.org/json-schema/-/json-schema-0.2.3.tgz", + "integrity": "sha1-tIDIkuWaLwWVTOcnvT8qTogvnhM=" }, "json-schema-traverse": { "version": "0.4.1", - "from": "json-schema-traverse@>=0.4.1 <0.5.0", - "resolved": "https://registry.npmjs.org/json-schema-traverse/-/json-schema-traverse-0.4.1.tgz" + "resolved": "https://registry.npmjs.org/json-schema-traverse/-/json-schema-traverse-0.4.1.tgz", + "integrity": "sha1-afaofZUTq4u4/mO9sJecRI5oRmA=" }, "json-stringify-safe": { "version": "5.0.1", - "from": "json-stringify-safe@5.0.1", - "resolved": "https://registry.npmjs.org/json-stringify-safe/-/json-stringify-safe-5.0.1.tgz" + "resolved": "https://registry.npmjs.org/json-stringify-safe/-/json-stringify-safe-5.0.1.tgz", + "integrity": "sha1-Epai1Y/UXxmg9s4B1lcB4sc1tus=" }, "jsprim": { "version": "1.4.1", - "from": "jsprim@>=1.2.2 <2.0.0", "resolved": "https://registry.npmjs.org/jsprim/-/jsprim-1.4.1.tgz", + "integrity": "sha1-MT5mvB5cwG5Di8G3SZwuXFastqI=", + "requires": { + "assert-plus": "1.0.0", + "extsprintf": "1.3.0", + "json-schema": "0.2.3", + "verror": "1.10.0" + }, "dependencies": { "assert-plus": { "version": "1.0.0", - "from": "assert-plus@1.0.0", - "resolved": "https://registry.npmjs.org/assert-plus/-/assert-plus-1.0.0.tgz" + "resolved": "https://registry.npmjs.org/assert-plus/-/assert-plus-1.0.0.tgz", + "integrity": "sha1-8S4PPF13sLHN2RRpQuTpbB5N1SU=" } } }, "jwa": { "version": "1.4.1", - "from": "jwa@>=1.4.1 <2.0.0", - "resolved": "https://registry.npmjs.org/jwa/-/jwa-1.4.1.tgz" + "resolved": "https://registry.npmjs.org/jwa/-/jwa-1.4.1.tgz", + "integrity": "sha1-dDwymFy56YZVUw1TZBtmyGRbA5o=", + "requires": { + "buffer-equal-constant-time": "1.0.1", + "ecdsa-sig-formatter": "1.0.11", + "safe-buffer": "^5.0.1" + } }, "jws": { "version": "3.2.2", - "from": "jws@>=3.1.5 <4.0.0", - "resolved": "https://registry.npmjs.org/jws/-/jws-3.2.2.tgz" + "resolved": "https://registry.npmjs.org/jws/-/jws-3.2.2.tgz", + "integrity": "sha1-ABCZ82OUaMlBQADpmZX6UvtHgwQ=", + "requires": { + "jwa": "^1.4.1", + "safe-buffer": "^5.0.1" + } }, "lodash": { "version": "4.17.15", - "from": "lodash@>=4.17.14 <5.0.0", - "resolved": "https://registry.npmjs.org/lodash/-/lodash-4.17.15.tgz" + "resolved": "https://registry.npmjs.org/lodash/-/lodash-4.17.15.tgz", + "integrity": "sha1-tEf2ZwoEVbv+7dETku/zMOoJdUg=" }, "lodash.defaults": { "version": "4.2.0", - "from": "lodash.defaults@>=4.2.0 <5.0.0", - "resolved": "https://registry.npmjs.org/lodash.defaults/-/lodash.defaults-4.2.0.tgz" + "resolved": "https://registry.npmjs.org/lodash.defaults/-/lodash.defaults-4.2.0.tgz", + "integrity": "sha1-0JF4cW/+pN3p5ft7N/bwgCJ0WAw=" }, "lodash.flatten": { "version": "4.4.0", - "from": "lodash.flatten@>=4.4.0 <5.0.0", - "resolved": "https://registry.npmjs.org/lodash.flatten/-/lodash.flatten-4.4.0.tgz" + "resolved": "https://registry.npmjs.org/lodash.flatten/-/lodash.flatten-4.4.0.tgz", + "integrity": "sha1-8xwiIlqWMtK7+OSt2+8kCqdlph8=" }, "lodash.pickby": { "version": "4.6.0", - "from": "lodash.pickby@>=4.6.0 <5.0.0", - "resolved": "https://registry.npmjs.org/lodash.pickby/-/lodash.pickby-4.6.0.tgz" + "resolved": "https://registry.npmjs.org/lodash.pickby/-/lodash.pickby-4.6.0.tgz", + "integrity": "sha1-feoh2MGNdwOifHBMFdO4SmfjOv8=" }, "logger-sharelatex": { "version": "1.7.0", - "from": "logger-sharelatex@1.7.0", "resolved": "https://registry.npmjs.org/logger-sharelatex/-/logger-sharelatex-1.7.0.tgz", + "integrity": "sha1-XuMje84im1rITZ7SLoXa6eI3/HQ=", + "requires": { + "bunyan": "1.8.12", + "raven": "1.1.3", + "request": "2.88.0" + }, "dependencies": { "bunyan": { "version": "1.8.12", - "from": "bunyan@1.8.12", - "resolved": "https://registry.npmjs.org/bunyan/-/bunyan-1.8.12.tgz" + "resolved": "https://registry.npmjs.org/bunyan/-/bunyan-1.8.12.tgz", + "integrity": "sha1-8VDw9nSKvdcq6uhPBEA74u8RN5c=", + "requires": { + "dtrace-provider": "~0.8", + "moment": "^2.10.6", + "mv": "~2", + "safe-json-stringify": "~1" + } }, "dtrace-provider": { "version": "0.8.7", - "from": "dtrace-provider@>=0.8.0 <0.9.0", "resolved": "https://registry.npmjs.org/dtrace-provider/-/dtrace-provider-0.8.7.tgz", - "optional": true + "integrity": "sha1-3JObTT4GIM/gwc2APQ0tftBP/QQ=", + "optional": true, + "requires": { + "nan": "^2.10.0" + } } } }, "lolex": { "version": "1.3.2", - "from": "lolex@1.3.2", "resolved": "https://registry.npmjs.org/lolex/-/lolex-1.3.2.tgz", + "integrity": "sha1-fD2mL/yzDw9agKJWbKJORdigHzE=", "dev": true }, "long": { "version": "4.0.0", - "from": "long@>=4.0.0 <5.0.0", - "resolved": "https://registry.npmjs.org/long/-/long-4.0.0.tgz" + "resolved": "https://registry.npmjs.org/long/-/long-4.0.0.tgz", + "integrity": "sha1-mntxz7fTYaGU6lVSQckvdGjVvyg=" }, "lru-cache": { "version": "5.1.1", - "from": "lru-cache@>=5.0.0 <6.0.0", - "resolved": "https://registry.npmjs.org/lru-cache/-/lru-cache-5.1.1.tgz" + "resolved": "https://registry.npmjs.org/lru-cache/-/lru-cache-5.1.1.tgz", + "integrity": "sha1-HaJ+ZxAnGUdpXa9oSOhH8B2EuSA=", + "requires": { + "yallist": "^3.0.2" + } }, "lsmod": { "version": "1.0.0", - "from": "lsmod@1.0.0", - "resolved": "https://registry.npmjs.org/lsmod/-/lsmod-1.0.0.tgz" + "resolved": "https://registry.npmjs.org/lsmod/-/lsmod-1.0.0.tgz", + "integrity": "sha1-mgD3bco26yP6BTUK/htYXUKZ5ks=" }, "lynx": { "version": "0.1.1", - "from": "lynx@>=0.1.1 <0.2.0", - "resolved": "https://registry.npmjs.org/lynx/-/lynx-0.1.1.tgz" + "resolved": "https://registry.npmjs.org/lynx/-/lynx-0.1.1.tgz", + "integrity": "sha1-Mxjc7xaQi4KG6Bisz9sxzXQkj50=", + "requires": { + "mersenne": "~0.0.3", + "statsd-parser": "~0.0.4" + } }, "media-typer": { "version": "0.3.0", - "from": "media-typer@0.3.0", - "resolved": "https://registry.npmjs.org/media-typer/-/media-typer-0.3.0.tgz" + "resolved": "https://registry.npmjs.org/media-typer/-/media-typer-0.3.0.tgz", + "integrity": "sha1-hxDXrwqmJvj/+hzgAWhUUmMlV0g=" }, "merge-descriptors": { "version": "1.0.1", - "from": "merge-descriptors@1.0.1", - "resolved": "https://registry.npmjs.org/merge-descriptors/-/merge-descriptors-1.0.1.tgz" + "resolved": "https://registry.npmjs.org/merge-descriptors/-/merge-descriptors-1.0.1.tgz", + "integrity": "sha1-sAqqVW3YtEVoFQ7J0blT8/kMu2E=" }, "mersenne": { "version": "0.0.4", - "from": "mersenne@>=0.0.3 <0.1.0", - "resolved": "https://registry.npmjs.org/mersenne/-/mersenne-0.0.4.tgz" + "resolved": "https://registry.npmjs.org/mersenne/-/mersenne-0.0.4.tgz", + "integrity": "sha1-QB/ex+whzbngPNPTAhOY2iGycIU=" }, "methods": { "version": "1.1.2", - "from": "methods@>=1.1.2 <1.2.0", - "resolved": "https://registry.npmjs.org/methods/-/methods-1.1.2.tgz" + "resolved": "https://registry.npmjs.org/methods/-/methods-1.1.2.tgz", + "integrity": "sha1-VSmk1nZUE07cxSZmVoNbD4Ua/O4=" }, "metrics-sharelatex": { "version": "2.2.0", - "from": "metrics-sharelatex@2.2.0", "resolved": "https://registry.npmjs.org/metrics-sharelatex/-/metrics-sharelatex-2.2.0.tgz", + "integrity": "sha1-RM9oy9FuUQYgfrZ+PvkAhaQWwqk=", + "requires": { + "@google-cloud/debug-agent": "^3.0.0", + "@google-cloud/profiler": "^0.2.3", + "@google-cloud/trace-agent": "^3.2.0", + "coffee-script": "1.6.0", + "lynx": "~0.1.1", + "prom-client": "^11.1.3", + "underscore": "~1.6.0" + }, "dependencies": { "underscore": { "version": "1.6.0", - "from": "underscore@>=1.6.0 <1.7.0", - "resolved": "https://registry.npmjs.org/underscore/-/underscore-1.6.0.tgz" + "resolved": "https://registry.npmjs.org/underscore/-/underscore-1.6.0.tgz", + "integrity": "sha1-izixDKze9jM3uLJOT/htRa6lKag=" } } }, "mime": { "version": "1.4.1", - "from": "mime@1.4.1", - "resolved": "https://registry.npmjs.org/mime/-/mime-1.4.1.tgz" + "resolved": "https://registry.npmjs.org/mime/-/mime-1.4.1.tgz", + "integrity": "sha1-Eh+evEnjdm8xGnbh+hyAA8SwOqY=" }, "mime-db": { "version": "1.33.0", - "from": "mime-db@>=1.33.0 <1.34.0", - "resolved": "https://registry.npmjs.org/mime-db/-/mime-db-1.33.0.tgz" + "resolved": "https://registry.npmjs.org/mime-db/-/mime-db-1.33.0.tgz", + "integrity": "sha1-o0kgUKXLm2NFBUHjnZeI0icng9s=" }, "mime-types": { "version": "2.1.18", - "from": "mime-types@>=2.1.18 <2.2.0", - "resolved": "https://registry.npmjs.org/mime-types/-/mime-types-2.1.18.tgz" + "resolved": "https://registry.npmjs.org/mime-types/-/mime-types-2.1.18.tgz", + "integrity": "sha1-bzI/YKg9ERRvgx/xH9ZuL+VQO7g=", + "requires": { + "mime-db": "~1.33.0" + } }, "minimatch": { "version": "3.0.4", - "from": "minimatch@2 || 3", - "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-3.0.4.tgz" + "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-3.0.4.tgz", + "integrity": "sha1-UWbihkV/AzBgZL5Ul+jbsMPTIIM=", + "requires": { + "brace-expansion": "^1.1.7" + } }, "minimist": { "version": "0.0.8", - "from": "minimist@0.0.8", - "resolved": "https://registry.npmjs.org/minimist/-/minimist-0.0.8.tgz" + "resolved": "https://registry.npmjs.org/minimist/-/minimist-0.0.8.tgz", + "integrity": "sha1-hX/Kv8M5fSYluCKCYuhqp6ARsF0=" }, "mkdirp": { "version": "0.5.1", - "from": "mkdirp@~0.5.1", - "resolved": "https://registry.npmjs.org/mkdirp/-/mkdirp-0.5.1.tgz" + "resolved": "https://registry.npmjs.org/mkdirp/-/mkdirp-0.5.1.tgz", + "integrity": "sha1-MAV0OOrGz3+MR2fzhkjWaX11yQM=", + "requires": { + "minimist": "0.0.8" + } }, "mocha": { "version": "4.1.0", - "from": "mocha@>=4.0.1 <5.0.0", "resolved": "https://registry.npmjs.org/mocha/-/mocha-4.1.0.tgz", + "integrity": "sha1-fYbPvPNcuCnidUwy4XNV7AUzh5Q=", "dev": true, + "requires": { + "browser-stdout": "1.3.0", + "commander": "2.11.0", + "debug": "3.1.0", + "diff": "3.3.1", + "escape-string-regexp": "1.0.5", + "glob": "7.1.2", + "growl": "1.10.3", + "he": "1.1.1", + "mkdirp": "0.5.1", + "supports-color": "4.4.0" + }, "dependencies": { "commander": { "version": "2.11.0", - "from": "commander@2.11.0", "resolved": "https://registry.npmjs.org/commander/-/commander-2.11.0.tgz", + "integrity": "sha1-FXFS/R56bI2YpbcVzzdt+SgARWM=", "dev": true }, "debug": { "version": "3.1.0", - "from": "debug@3.1.0", "resolved": "https://registry.npmjs.org/debug/-/debug-3.1.0.tgz", - "dev": true + "integrity": "sha1-W7WgZyYotkFJVmuhaBnmFRjGcmE=", + "dev": true, + "requires": { + "ms": "2.0.0" + } }, "diff": { "version": "3.3.1", - "from": "diff@3.3.1", "resolved": "https://registry.npmjs.org/diff/-/diff-3.3.1.tgz", + "integrity": "sha1-qoVnpu7QPFMfyJ0/cRzQ5SWd7HU=", "dev": true }, "glob": { "version": "7.1.2", - "from": "glob@7.1.2", "resolved": "https://registry.npmjs.org/glob/-/glob-7.1.2.tgz", - "dev": true + "integrity": "sha1-wZyd+aAocC1nhhI4SmVSQExjbRU=", + "dev": true, + "requires": { + "fs.realpath": "^1.0.0", + "inflight": "^1.0.4", + "inherits": "2", + "minimatch": "^3.0.4", + "once": "^1.3.0", + "path-is-absolute": "^1.0.0" + } }, "growl": { "version": "1.10.3", - "from": "growl@1.10.3", "resolved": "https://registry.npmjs.org/growl/-/growl-1.10.3.tgz", + "integrity": "sha1-GSa6kM8+3+KttJJ/WIC8IsZseQ8=", "dev": true }, "has-flag": { "version": "2.0.0", - "from": "has-flag@>=2.0.0 <3.0.0", "resolved": "https://registry.npmjs.org/has-flag/-/has-flag-2.0.0.tgz", + "integrity": "sha1-6CB68cx7MNRGzHC3NLXovhj4jVE=", "dev": true }, "supports-color": { "version": "4.4.0", - "from": "supports-color@4.4.0", "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-4.4.0.tgz", - "dev": true + "integrity": "sha1-iD992rwWUUKyphQn8zUt7RldGj4=", + "dev": true, + "requires": { + "has-flag": "^2.0.0" + } } } }, "module-details-from-path": { "version": "1.0.3", - "from": "module-details-from-path@>=1.0.3 <2.0.0", - "resolved": "https://registry.npmjs.org/module-details-from-path/-/module-details-from-path-1.0.3.tgz" + "resolved": "https://registry.npmjs.org/module-details-from-path/-/module-details-from-path-1.0.3.tgz", + "integrity": "sha1-EUyUlnPiqKNenTV4hSeqN7Z52is=" }, "moment": { "version": "2.24.0", - "from": "moment@>=2.10.6 <3.0.0", "resolved": "https://registry.npmjs.org/moment/-/moment-2.24.0.tgz", + "integrity": "sha1-DQVdU/UFKqZTyfbraLtdEr9cK1s=", "optional": true }, "ms": { "version": "2.0.0", - "from": "ms@2.0.0", - "resolved": "https://registry.npmjs.org/ms/-/ms-2.0.0.tgz" + "resolved": "https://registry.npmjs.org/ms/-/ms-2.0.0.tgz", + "integrity": "sha1-VgiurfwAvmwpAd9fmGF4jeDVl8g=" }, "mv": { "version": "2.1.1", - "from": "mv@~2", "resolved": "https://registry.npmjs.org/mv/-/mv-2.1.1.tgz", - "optional": true + "integrity": "sha1-rmzg1vbV4KT32JN5jQPB6pVZtqI=", + "optional": true, + "requires": { + "mkdirp": "~0.5.1", + "ncp": "~2.0.0", + "rimraf": "~2.4.0" + } }, "nan": { "version": "2.12.1", - "from": "nan@>=2.0.8 <3.0.0", - "resolved": "https://registry.npmjs.org/nan/-/nan-2.12.1.tgz" + "resolved": "https://registry.npmjs.org/nan/-/nan-2.12.1.tgz", + "integrity": "sha1-exqhk+mqhgV+PHu9CsRI53CSVVI=" }, "native-or-bluebird": { "version": "1.1.2", - "from": "native-or-bluebird@>=1.1.2 <1.2.0", "resolved": "https://registry.npmjs.org/native-or-bluebird/-/native-or-bluebird-1.1.2.tgz", + "integrity": "sha1-OSHhECMtHreQ89rGG7NwUxx9NW4=", "dev": true }, "ncp": { "version": "2.0.0", - "from": "ncp@~2.0.0", "resolved": "https://registry.npmjs.org/ncp/-/ncp-2.0.0.tgz", + "integrity": "sha1-GVoh1sRuNh0vsSgbo4uR6d9727M=", "optional": true }, "negotiator": { "version": "0.6.1", - "from": "negotiator@0.6.1", - "resolved": "https://registry.npmjs.org/negotiator/-/negotiator-0.6.1.tgz" + "resolved": "https://registry.npmjs.org/negotiator/-/negotiator-0.6.1.tgz", + "integrity": "sha1-KzJxhOiZIQEXeyhWP7XnECrNDKk=" }, "node-fetch": { "version": "2.6.0", - "from": "node-fetch@>=2.3.0 <3.0.0", - "resolved": "https://registry.npmjs.org/node-fetch/-/node-fetch-2.6.0.tgz" + "resolved": "https://registry.npmjs.org/node-fetch/-/node-fetch-2.6.0.tgz", + "integrity": "sha1-5jNFY4bUqlWGP2dqerDaqP3ssP0=" }, "node-forge": { "version": "0.8.4", - "from": "node-forge@>=0.8.0 <0.9.0", - "resolved": "https://registry.npmjs.org/node-forge/-/node-forge-0.8.4.tgz" + "resolved": "https://registry.npmjs.org/node-forge/-/node-forge-0.8.4.tgz", + "integrity": "sha1-1nOGYrZhvhnicR7wGqOxghLxMDA=" }, "oauth-sign": { "version": "0.9.0", - "from": "oauth-sign@>=0.9.0 <0.10.0", - "resolved": "https://registry.npmjs.org/oauth-sign/-/oauth-sign-0.9.0.tgz" + "resolved": "https://registry.npmjs.org/oauth-sign/-/oauth-sign-0.9.0.tgz", + "integrity": "sha1-R6ewFrqmi1+g7PPe4IqFxnmsZFU=" }, "object-keys": { "version": "1.1.1", - "from": "object-keys@>=1.0.12 <2.0.0", "resolved": "https://registry.npmjs.org/object-keys/-/object-keys-1.1.1.tgz", + "integrity": "sha1-HEfyct8nfzsdrwYWd9nILiMixg4=", "dev": true }, "object.entries": { "version": "1.1.0", - "from": "object.entries@>=1.1.0 <2.0.0", "resolved": "https://registry.npmjs.org/object.entries/-/object.entries-1.1.0.tgz", - "dev": true + "integrity": "sha1-ICT8bWuiRq7ji9sP/Vz7zzcbdRk=", + "dev": true, + "requires": { + "define-properties": "^1.1.3", + "es-abstract": "^1.12.0", + "function-bind": "^1.1.1", + "has": "^1.0.3" + } }, "on-finished": { "version": "2.3.0", - "from": "on-finished@>=2.3.0 <2.4.0", - "resolved": "https://registry.npmjs.org/on-finished/-/on-finished-2.3.0.tgz" + "resolved": "https://registry.npmjs.org/on-finished/-/on-finished-2.3.0.tgz", + "integrity": "sha1-IPEzZIGwg811M3mSoWlxqi2QaUc=", + "requires": { + "ee-first": "1.1.1" + } }, "on-headers": { "version": "1.0.1", - "from": "on-headers@>=1.0.1 <1.1.0", - "resolved": "https://registry.npmjs.org/on-headers/-/on-headers-1.0.1.tgz" + "resolved": "https://registry.npmjs.org/on-headers/-/on-headers-1.0.1.tgz", + "integrity": "sha1-ko9dD0cNSTQmUepnlLCFfBAGk/c=" }, "once": { "version": "1.4.0", - "from": "once@^1.3.0", - "resolved": "https://registry.npmjs.org/once/-/once-1.4.0.tgz" + "resolved": "https://registry.npmjs.org/once/-/once-1.4.0.tgz", + "integrity": "sha1-WDsap3WWHUsROsF9nFC6753Xa9E=", + "requires": { + "wrappy": "1" + } }, "options": { "version": "0.0.6", - "from": "options@>=0.0.5", - "resolved": "https://registry.npmjs.org/options/-/options-0.0.6.tgz" + "resolved": "https://registry.npmjs.org/options/-/options-0.0.6.tgz", + "integrity": "sha1-7CLTEoBrtT5zF3Pnza788cZDEo8=" }, "p-limit": { "version": "2.2.0", - "from": "p-limit@>=2.2.0 <3.0.0", - "resolved": "https://registry.npmjs.org/p-limit/-/p-limit-2.2.0.tgz" + "resolved": "https://registry.npmjs.org/p-limit/-/p-limit-2.2.0.tgz", + "integrity": "sha1-QXyZQeYCepq8ulCS3SkE4lW1+8I=", + "requires": { + "p-try": "^2.0.0" + } }, "p-try": { "version": "2.2.0", - "from": "p-try@>=2.0.0 <3.0.0", - "resolved": "https://registry.npmjs.org/p-try/-/p-try-2.2.0.tgz" + "resolved": "https://registry.npmjs.org/p-try/-/p-try-2.2.0.tgz", + "integrity": "sha1-yyhoVA4xPWHeWPr741zpAE1VQOY=" }, "parse-duration": { "version": "0.1.1", - "from": "parse-duration@>=0.1.1 <0.2.0", - "resolved": "https://registry.npmjs.org/parse-duration/-/parse-duration-0.1.1.tgz" + "resolved": "https://registry.npmjs.org/parse-duration/-/parse-duration-0.1.1.tgz", + "integrity": "sha1-ExFN3JiRwezSgANiRFVN5DZHoiY=" }, "parse-ms": { "version": "2.1.0", - "from": "parse-ms@>=2.0.0 <3.0.0", - "resolved": "https://registry.npmjs.org/parse-ms/-/parse-ms-2.1.0.tgz" + "resolved": "https://registry.npmjs.org/parse-ms/-/parse-ms-2.1.0.tgz", + "integrity": "sha1-NIVlp1PUOR+lJAKZVrFyy3dTCX0=" }, "parseurl": { "version": "1.3.2", - "from": "parseurl@>=1.3.2 <1.4.0", - "resolved": "https://registry.npmjs.org/parseurl/-/parseurl-1.3.2.tgz" + "resolved": "https://registry.npmjs.org/parseurl/-/parseurl-1.3.2.tgz", + "integrity": "sha1-/CidTtiZMRlGDBViUyYs3I3mW/M=" }, "path-is-absolute": { "version": "1.0.1", - "from": "path-is-absolute@^1.0.0", - "resolved": "https://registry.npmjs.org/path-is-absolute/-/path-is-absolute-1.0.1.tgz" + "resolved": "https://registry.npmjs.org/path-is-absolute/-/path-is-absolute-1.0.1.tgz", + "integrity": "sha1-F0uSaHNVNP+8es5r9TpanhtcX18=" }, "path-parse": { "version": "1.0.6", - "from": "path-parse@>=1.0.5 <2.0.0", - "resolved": "https://registry.npmjs.org/path-parse/-/path-parse-1.0.6.tgz" + "resolved": "https://registry.npmjs.org/path-parse/-/path-parse-1.0.6.tgz", + "integrity": "sha1-1i27VnlAXXLEc37FhgDp3c8G0kw=" }, "path-to-regexp": { "version": "0.1.7", - "from": "path-to-regexp@0.1.7", - "resolved": "https://registry.npmjs.org/path-to-regexp/-/path-to-regexp-0.1.7.tgz" + "resolved": "https://registry.npmjs.org/path-to-regexp/-/path-to-regexp-0.1.7.tgz", + "integrity": "sha1-32BBeABfUi8V60SQ5yR6G/qmf4w=" }, "performance-now": { "version": "2.1.0", - "from": "performance-now@>=2.1.0 <3.0.0", - "resolved": "https://registry.npmjs.org/performance-now/-/performance-now-2.1.0.tgz" + "resolved": "https://registry.npmjs.org/performance-now/-/performance-now-2.1.0.tgz", + "integrity": "sha1-Ywn04OX6kT7BxpMHrjZLSzd8nns=" }, "pify": { "version": "4.0.1", - "from": "pify@>=4.0.0 <5.0.0", - "resolved": "https://registry.npmjs.org/pify/-/pify-4.0.1.tgz" + "resolved": "https://registry.npmjs.org/pify/-/pify-4.0.1.tgz", + "integrity": "sha1-SyzSXFDVmHNcUCkiJP2MbfQeMjE=" }, "policyfile": { "version": "0.0.4", - "from": "policyfile@0.0.4", - "resolved": "https://registry.npmjs.org/policyfile/-/policyfile-0.0.4.tgz" + "resolved": "https://registry.npmjs.org/policyfile/-/policyfile-0.0.4.tgz", + "integrity": "sha1-1rgurZiueeviKOLa9ZAzEeyYLk0=" }, "pretty-ms": { "version": "4.0.0", - "from": "pretty-ms@>=4.0.0 <5.0.0", - "resolved": "https://registry.npmjs.org/pretty-ms/-/pretty-ms-4.0.0.tgz" + "resolved": "https://registry.npmjs.org/pretty-ms/-/pretty-ms-4.0.0.tgz", + "integrity": "sha1-Mbr0G5T9AiJwmKqgO9YmCOsNbpI=", + "requires": { + "parse-ms": "^2.0.0" + } }, "process-nextick-args": { "version": "2.0.0", - "from": "process-nextick-args@>=2.0.0 <2.1.0", - "resolved": "https://registry.npmjs.org/process-nextick-args/-/process-nextick-args-2.0.0.tgz" + "resolved": "https://registry.npmjs.org/process-nextick-args/-/process-nextick-args-2.0.0.tgz", + "integrity": "sha1-o31zL0JxtKsa0HDTVQjoKQeI/6o=" }, "prom-client": { "version": "11.5.1", - "from": "prom-client@>=11.1.3 <12.0.0", - "resolved": "https://registry.npmjs.org/prom-client/-/prom-client-11.5.1.tgz" + "resolved": "https://registry.npmjs.org/prom-client/-/prom-client-11.5.1.tgz", + "integrity": "sha1-FcZsrN7EUwELz68EEJvMNOa92pw=", + "requires": { + "tdigest": "^0.1.1" + } }, "protobufjs": { "version": "6.8.8", - "from": "protobufjs@>=6.8.6 <6.9.0", "resolved": "https://registry.npmjs.org/protobufjs/-/protobufjs-6.8.8.tgz", + "integrity": "sha1-yLTxKC/XqQ5vWxCe0RyEr4KQjnw=", + "requires": { + "@protobufjs/aspromise": "^1.1.2", + "@protobufjs/base64": "^1.1.2", + "@protobufjs/codegen": "^2.0.4", + "@protobufjs/eventemitter": "^1.1.0", + "@protobufjs/fetch": "^1.1.0", + "@protobufjs/float": "^1.0.2", + "@protobufjs/inquire": "^1.1.0", + "@protobufjs/path": "^1.1.2", + "@protobufjs/pool": "^1.1.0", + "@protobufjs/utf8": "^1.1.0", + "@types/long": "^4.0.0", + "@types/node": "^10.1.0", + "long": "^4.0.0" + }, "dependencies": { "@types/node": { "version": "10.14.9", - "from": "@types/node@>=10.1.0 <11.0.0", - "resolved": "https://registry.npmjs.org/@types/node/-/node-10.14.9.tgz" + "resolved": "https://registry.npmjs.org/@types/node/-/node-10.14.9.tgz", + "integrity": "sha1-Lo1ngDnSeUPOU6GRM4YTMif9kGY=" } } }, "proxy-addr": { "version": "2.0.3", - "from": "proxy-addr@>=2.0.3 <2.1.0", - "resolved": "https://registry.npmjs.org/proxy-addr/-/proxy-addr-2.0.3.tgz" + "resolved": "https://registry.npmjs.org/proxy-addr/-/proxy-addr-2.0.3.tgz", + "integrity": "sha1-NV8mJQWmIWRrMTCnKOtkfiIFU0E=", + "requires": { + "forwarded": "~0.1.2", + "ipaddr.js": "1.6.0" + } }, "psl": { "version": "1.1.31", - "from": "psl@>=1.1.24 <2.0.0", - "resolved": "https://registry.npmjs.org/psl/-/psl-1.1.31.tgz" + "resolved": "https://registry.npmjs.org/psl/-/psl-1.1.31.tgz", + "integrity": "sha1-6aqG0BAbWxBcvpOsa3hM1UcnYYQ=" }, "punycode": { "version": "1.4.1", - "from": "punycode@>=1.4.1 <2.0.0", - "resolved": "https://registry.npmjs.org/punycode/-/punycode-1.4.1.tgz" + "resolved": "https://registry.npmjs.org/punycode/-/punycode-1.4.1.tgz", + "integrity": "sha1-wNWmOycYgArY4esPpSachN1BhF4=" }, "q": { "version": "0.9.2", - "from": "q@0.9.2", - "resolved": "https://registry.npmjs.org/q/-/q-0.9.2.tgz" + "resolved": "https://registry.npmjs.org/q/-/q-0.9.2.tgz", + "integrity": "sha1-I8BsRsgTKGFqrhaNPuI6Vr1D2vY=" }, "qs": { "version": "6.5.2", - "from": "qs@6.5.2", - "resolved": "https://registry.npmjs.org/qs/-/qs-6.5.2.tgz" + "resolved": "https://registry.npmjs.org/qs/-/qs-6.5.2.tgz", + "integrity": "sha1-yzroBuh0BERYTvFUzo7pjUA/PjY=" }, "random-bytes": { "version": "1.0.0", - "from": "random-bytes@>=1.0.0 <1.1.0", - "resolved": "https://registry.npmjs.org/random-bytes/-/random-bytes-1.0.0.tgz" + "resolved": "https://registry.npmjs.org/random-bytes/-/random-bytes-1.0.0.tgz", + "integrity": "sha1-T2ih3Arli9P7lYSMMDJNt11kNgs=" }, "range-parser": { "version": "1.2.0", - "from": "range-parser@>=1.2.0 <1.3.0", - "resolved": "https://registry.npmjs.org/range-parser/-/range-parser-1.2.0.tgz" + "resolved": "https://registry.npmjs.org/range-parser/-/range-parser-1.2.0.tgz", + "integrity": "sha1-9JvmtIeJTdxA3MlKMi9hEJLgDV4=" }, "raven": { "version": "1.1.3", - "from": "raven@1.1.3", - "resolved": "https://registry.npmjs.org/raven/-/raven-1.1.3.tgz" + "resolved": "https://registry.npmjs.org/raven/-/raven-1.1.3.tgz", + "integrity": "sha1-QnPBrm005CMPUbLAEEGjK5Iygio=", + "requires": { + "cookie": "0.3.1", + "json-stringify-safe": "5.0.1", + "lsmod": "1.0.0", + "stack-trace": "0.0.9", + "uuid": "3.0.0" + } }, "raw-body": { "version": "2.3.3", - "from": "raw-body@2.3.3", - "resolved": "https://registry.npmjs.org/raw-body/-/raw-body-2.3.3.tgz" + "resolved": "https://registry.npmjs.org/raw-body/-/raw-body-2.3.3.tgz", + "integrity": "sha1-GzJOzmtXBuFThVvBFIxlu39uoMM=", + "requires": { + "bytes": "3.0.0", + "http-errors": "1.6.3", + "iconv-lite": "0.4.23", + "unpipe": "1.0.0" + } }, "readable-stream": { "version": "2.3.6", - "from": "readable-stream@>=2.0.0 <3.0.0", - "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-2.3.6.tgz" + "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-2.3.6.tgz", + "integrity": "sha1-sRwn2IuP8fvgcGQ8+UsMea4bCq8=", + "requires": { + "core-util-is": "~1.0.0", + "inherits": "~2.0.3", + "isarray": "~1.0.0", + "process-nextick-args": "~2.0.0", + "safe-buffer": "~5.1.1", + "string_decoder": "~1.1.1", + "util-deprecate": "~1.0.1" + } }, "redis": { "version": "0.12.1", - "from": "redis@>=0.12.1 <0.13.0", - "resolved": "https://registry.npmjs.org/redis/-/redis-0.12.1.tgz" + "resolved": "https://registry.npmjs.org/redis/-/redis-0.12.1.tgz", + "integrity": "sha1-ZN92rQ/IrOuuvSoGReikj6xJGF4=" }, "redis-commands": { "version": "1.5.0", - "from": "redis-commands@1.5.0", - "resolved": "https://registry.npmjs.org/redis-commands/-/redis-commands-1.5.0.tgz" + "resolved": "https://registry.npmjs.org/redis-commands/-/redis-commands-1.5.0.tgz", + "integrity": "sha1-gNLiBpj+aI8icSf/nlFkp90X54U=" }, "redis-errors": { "version": "1.2.0", - "from": "redis-errors@>=1.2.0 <2.0.0", - "resolved": "https://registry.npmjs.org/redis-errors/-/redis-errors-1.2.0.tgz" + "resolved": "https://registry.npmjs.org/redis-errors/-/redis-errors-1.2.0.tgz", + "integrity": "sha1-62LSrbFeTq9GEMBK/hUpOEJQq60=" }, "redis-parser": { "version": "3.0.0", - "from": "redis-parser@>=3.0.0 <4.0.0", - "resolved": "https://registry.npmjs.org/redis-parser/-/redis-parser-3.0.0.tgz" + "resolved": "https://registry.npmjs.org/redis-parser/-/redis-parser-3.0.0.tgz", + "integrity": "sha1-tm2CjNyv5rS4pCin3vTGvKwxyLQ=", + "requires": { + "redis-errors": "^1.0.0" + } }, "redis-sentinel": { "version": "0.1.1", - "from": "redis-sentinel@0.1.1", "resolved": "https://registry.npmjs.org/redis-sentinel/-/redis-sentinel-0.1.1.tgz", + "integrity": "sha1-Vj3TQduZMgMfSX+v3Td+hkj/s+U=", + "requires": { + "q": "0.9.2", + "redis": "0.11.x" + }, "dependencies": { "redis": { "version": "0.11.0", - "from": "redis@>=0.11.0 <0.12.0", - "resolved": "https://registry.npmjs.org/redis/-/redis-0.11.0.tgz" + "resolved": "https://registry.npmjs.org/redis/-/redis-0.11.0.tgz", + "integrity": "sha1-/cAdSrTL5LO7LLKByP5WnDhX9XE=" } } }, "redis-sharelatex": { "version": "1.0.11", - "from": "redis-sharelatex@1.0.11", "resolved": "https://registry.npmjs.org/redis-sharelatex/-/redis-sharelatex-1.0.11.tgz", + "integrity": "sha1-fcTlS2/FHrsh3WjJcOgdIMPEaMM=", + "requires": { + "async": "^2.5.0", + "coffee-script": "1.8.0", + "ioredis": "~4.14.1", + "redis-sentinel": "0.1.1", + "underscore": "1.7.0" + }, "dependencies": { "async": { "version": "2.6.3", - "from": "async@>=2.5.0 <3.0.0", - "resolved": "https://registry.npmjs.org/async/-/async-2.6.3.tgz" + "resolved": "https://registry.npmjs.org/async/-/async-2.6.3.tgz", + "integrity": "sha1-1yYl4jRKNlbjo61Pp0n6gymdgv8=", + "requires": { + "lodash": "^4.17.14" + } }, "coffee-script": { "version": "1.8.0", - "from": "coffee-script@https://registry.npmjs.org/coffee-script/-/coffee-script-1.8.0.tgz", - "resolved": "https://registry.npmjs.org/coffee-script/-/coffee-script-1.8.0.tgz" + "resolved": "https://registry.npmjs.org/coffee-script/-/coffee-script-1.8.0.tgz", + "integrity": "sha1-nJ8dK0pSoADe0Vtll5FwNkgmPB0=", + "requires": { + "mkdirp": "~0.3.5" + } }, "mkdirp": { "version": "0.3.5", - "from": "mkdirp@https://registry.npmjs.org/mkdirp/-/mkdirp-0.3.5.tgz", - "resolved": "https://registry.npmjs.org/mkdirp/-/mkdirp-0.3.5.tgz" + "resolved": "https://registry.npmjs.org/mkdirp/-/mkdirp-0.3.5.tgz", + "integrity": "sha1-3j5fiWHIjHh+4TaN+EmsRBPsqNc=" } } }, "request": { "version": "2.88.0", - "from": "request@2.88.0", "resolved": "https://registry.npmjs.org/request/-/request-2.88.0.tgz", + "integrity": "sha1-nC/KT301tZLv5Xx/ClXoEFIST+8=", + "requires": { + "aws-sign2": "~0.7.0", + "aws4": "^1.8.0", + "caseless": "~0.12.0", + "combined-stream": "~1.0.6", + "extend": "~3.0.2", + "forever-agent": "~0.6.1", + "form-data": "~2.3.2", + "har-validator": "~5.1.0", + "http-signature": "~1.2.0", + "is-typedarray": "~1.0.0", + "isstream": "~0.1.2", + "json-stringify-safe": "~5.0.1", + "mime-types": "~2.1.19", + "oauth-sign": "~0.9.0", + "performance-now": "^2.1.0", + "qs": "~6.5.2", + "safe-buffer": "^5.1.2", + "tough-cookie": "~2.4.3", + "tunnel-agent": "^0.6.0", + "uuid": "^3.3.2" + }, "dependencies": { "mime-db": { "version": "1.38.0", - "from": "mime-db@>=1.38.0 <1.39.0", - "resolved": "https://registry.npmjs.org/mime-db/-/mime-db-1.38.0.tgz" + "resolved": "https://registry.npmjs.org/mime-db/-/mime-db-1.38.0.tgz", + "integrity": "sha1-GiqrFtqesWe0nG5N8tnGjWPY4q0=" }, "mime-types": { "version": "2.1.22", - "from": "mime-types@>=2.1.19 <2.2.0", - "resolved": "https://registry.npmjs.org/mime-types/-/mime-types-2.1.22.tgz" + "resolved": "https://registry.npmjs.org/mime-types/-/mime-types-2.1.22.tgz", + "integrity": "sha1-/ms1WhkJJqt2mMmgVWoRGZshmb0=", + "requires": { + "mime-db": "~1.38.0" + } }, "safe-buffer": { "version": "5.1.2", - "from": "safe-buffer@>=5.1.2 <6.0.0", - "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.1.2.tgz" + "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.1.2.tgz", + "integrity": "sha1-mR7GnSluAxN0fVm9/St0XDX4go0=" }, "uuid": { "version": "3.3.2", - "from": "uuid@>=3.3.2 <4.0.0", - "resolved": "https://registry.npmjs.org/uuid/-/uuid-3.3.2.tgz" + "resolved": "https://registry.npmjs.org/uuid/-/uuid-3.3.2.tgz", + "integrity": "sha1-G0r0lV6zB3xQHCOHL8ZROBFYcTE=" } } }, "require-in-the-middle": { "version": "4.0.0", - "from": "require-in-the-middle@>=4.0.0 <5.0.0", "resolved": "https://registry.npmjs.org/require-in-the-middle/-/require-in-the-middle-4.0.0.tgz", + "integrity": "sha1-PHUoik7EgM30S8d950T4q+WFQFs=", + "requires": { + "debug": "^4.1.1", + "module-details-from-path": "^1.0.3", + "resolve": "^1.10.0" + }, "dependencies": { "debug": { "version": "4.1.1", - "from": "debug@>=4.1.1 <5.0.0", - "resolved": "https://registry.npmjs.org/debug/-/debug-4.1.1.tgz" + "resolved": "https://registry.npmjs.org/debug/-/debug-4.1.1.tgz", + "integrity": "sha1-O3ImAlUQnGtYnO4FDx1RYTlmR5E=", + "requires": { + "ms": "^2.1.1" + } }, "ms": { "version": "2.1.2", - "from": "ms@^2.1.1", - "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.2.tgz" + "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.2.tgz", + "integrity": "sha1-0J0fNXtEP0kzgqjrPM0YOHKuYAk=" } } }, "require-like": { "version": "0.1.2", - "from": "require-like@0.1.2", "resolved": "https://registry.npmjs.org/require-like/-/require-like-0.1.2.tgz", + "integrity": "sha1-rW8wwTvs15cBDEaK+ndcDAprR/o=", "dev": true }, "resolve": { "version": "1.11.0", - "from": "resolve@>=1.10.0 <2.0.0", - "resolved": "https://registry.npmjs.org/resolve/-/resolve-1.11.0.tgz" + "resolved": "https://registry.npmjs.org/resolve/-/resolve-1.11.0.tgz", + "integrity": "sha1-QBSHC6KWF2uGND1Qtg87UGCc4jI=", + "requires": { + "path-parse": "^1.0.6" + } }, "retry-axios": { "version": "0.3.2", - "from": "retry-axios@>=0.3.2 <0.4.0", - "resolved": "https://registry.npmjs.org/retry-axios/-/retry-axios-0.3.2.tgz" + "resolved": "https://registry.npmjs.org/retry-axios/-/retry-axios-0.3.2.tgz", + "integrity": "sha1-V1fID1hbTMTEmGqi/9R6YMbTXhM=" }, "retry-request": { "version": "4.0.0", - "from": "retry-request@>=4.0.0 <5.0.0", - "resolved": "https://registry.npmjs.org/retry-request/-/retry-request-4.0.0.tgz" + "resolved": "https://registry.npmjs.org/retry-request/-/retry-request-4.0.0.tgz", + "integrity": "sha1-XDZhZiebPhDp16oTJ0RnoFy2kpA=", + "requires": { + "through2": "^2.0.0" + } }, "rimraf": { "version": "2.4.5", - "from": "rimraf@~2.4.0", "resolved": "https://registry.npmjs.org/rimraf/-/rimraf-2.4.5.tgz", - "optional": true + "integrity": "sha1-7nEM5dk6j9uFb7Xqj/Di11k0sto=", + "optional": true, + "requires": { + "glob": "^6.0.1" + } }, "safe-buffer": { "version": "5.1.1", - "from": "safe-buffer@5.1.1", - "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.1.1.tgz" + "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.1.1.tgz", + "integrity": "sha1-iTMSr2myEj3vcfV4iQAWce6yyFM=" }, "safe-json-stringify": { "version": "1.2.0", - "from": "safe-json-stringify@>=1.0.0 <2.0.0", "resolved": "https://registry.npmjs.org/safe-json-stringify/-/safe-json-stringify-1.2.0.tgz", + "integrity": "sha1-NW5EvJjx+TzkXfFLzXwBzahuCv0=", "optional": true }, "safer-buffer": { "version": "2.1.2", - "from": "safer-buffer@>=2.1.2 <3.0.0", - "resolved": "https://registry.npmjs.org/safer-buffer/-/safer-buffer-2.1.2.tgz" + "resolved": "https://registry.npmjs.org/safer-buffer/-/safer-buffer-2.1.2.tgz", + "integrity": "sha1-RPoWGwGHuVSd2Eu5GAL5vYOFzWo=" }, "samsam": { "version": "1.1.2", - "from": "samsam@1.1.2", "resolved": "https://registry.npmjs.org/samsam/-/samsam-1.1.2.tgz", + "integrity": "sha1-vsEf3IOp/aBjQBIQ5AF2wwJNFWc=", "dev": true }, "sandboxed-module": { "version": "0.3.0", - "from": "sandboxed-module@>=0.3.0 <0.4.0", "resolved": "https://registry.npmjs.org/sandboxed-module/-/sandboxed-module-0.3.0.tgz", + "integrity": "sha1-8fvvvYCaT2kHO9B8rm/H2y6vX2o=", "dev": true, + "requires": { + "require-like": "0.1.2", + "stack-trace": "0.0.6" + }, "dependencies": { "stack-trace": { "version": "0.0.6", - "from": "stack-trace@0.0.6", "resolved": "https://registry.npmjs.org/stack-trace/-/stack-trace-0.0.6.tgz", + "integrity": "sha1-HnGb1qJin/CcGJ4Xqe+QKpT8XbA=", "dev": true } } }, "semver": { "version": "6.1.1", - "from": "semver@>=6.0.0 <7.0.0", - "resolved": "https://registry.npmjs.org/semver/-/semver-6.1.1.tgz" + "resolved": "https://registry.npmjs.org/semver/-/semver-6.1.1.tgz", + "integrity": "sha1-U/U9qbMLIQPNTxXqs6GOy8shDJs=" }, "send": { "version": "0.16.2", - "from": "send@0.16.2", "resolved": "https://registry.npmjs.org/send/-/send-0.16.2.tgz", + "integrity": "sha1-bsyh4PjBVtFBWXVZhI32RzCmu8E=", + "requires": { + "debug": "2.6.9", + "depd": "~1.1.2", + "destroy": "~1.0.4", + "encodeurl": "~1.0.2", + "escape-html": "~1.0.3", + "etag": "~1.8.1", + "fresh": "0.5.2", + "http-errors": "~1.6.2", + "mime": "1.4.1", + "ms": "2.0.0", + "on-finished": "~2.3.0", + "range-parser": "~1.2.0", + "statuses": "~1.4.0" + }, "dependencies": { "statuses": { "version": "1.4.0", - "from": "statuses@>=1.4.0 <1.5.0", - "resolved": "https://registry.npmjs.org/statuses/-/statuses-1.4.0.tgz" + "resolved": "https://registry.npmjs.org/statuses/-/statuses-1.4.0.tgz", + "integrity": "sha1-u3PURtonlhBu/MG2AaJT1sRr0Ic=" } } }, "serve-static": { "version": "1.13.2", - "from": "serve-static@1.13.2", - "resolved": "https://registry.npmjs.org/serve-static/-/serve-static-1.13.2.tgz" + "resolved": "https://registry.npmjs.org/serve-static/-/serve-static-1.13.2.tgz", + "integrity": "sha1-CV6Ecv1bRiN9tQzkhqQ/S4bGzsE=", + "requires": { + "encodeurl": "~1.0.2", + "escape-html": "~1.0.3", + "parseurl": "~1.3.2", + "send": "0.16.2" + } }, "session.socket.io": { "version": "0.1.6", - "from": "session.socket.io@>=0.1.6 <0.2.0", - "resolved": "https://registry.npmjs.org/session.socket.io/-/session.socket.io-0.1.6.tgz" + "resolved": "https://registry.npmjs.org/session.socket.io/-/session.socket.io-0.1.6.tgz", + "integrity": "sha1-vh7sJAYJWgP4dw6ozN5s8SYBxdA=" }, "setprototypeof": { "version": "1.1.0", - "from": "setprototypeof@1.1.0", - "resolved": "https://registry.npmjs.org/setprototypeof/-/setprototypeof-1.1.0.tgz" + "resolved": "https://registry.npmjs.org/setprototypeof/-/setprototypeof-1.1.0.tgz", + "integrity": "sha1-0L2FU2iHtv58DYGMuWLZ2RxU5lY=" }, "settings-sharelatex": { "version": "1.1.0", - "from": "settings-sharelatex@1.1.0", "resolved": "https://registry.npmjs.org/settings-sharelatex/-/settings-sharelatex-1.1.0.tgz", + "integrity": "sha1-Tv4vUpPbjxwVlnEEx5BfqHD/mS0=", + "requires": { + "coffee-script": "1.6.0" + }, "dependencies": { "coffee-script": { "version": "1.6.0", - "from": "coffee-script@1.6.0", - "resolved": "https://registry.npmjs.org/coffee-script/-/coffee-script-1.6.0.tgz" + "resolved": "https://registry.npmjs.org/coffee-script/-/coffee-script-1.6.0.tgz", + "integrity": "sha1-gIs5bhEPU9AhoZpO8fZb4OjjX6M=" } } }, "shimmer": { "version": "1.2.1", - "from": "shimmer@>=1.2.0 <2.0.0", - "resolved": "https://registry.npmjs.org/shimmer/-/shimmer-1.2.1.tgz" + "resolved": "https://registry.npmjs.org/shimmer/-/shimmer-1.2.1.tgz", + "integrity": "sha1-YQhZ994ye1h+/r9QH7QxF/mv8zc=" }, "sinon": { "version": "1.17.7", - "from": "sinon@1.17.7", "resolved": "https://registry.npmjs.org/sinon/-/sinon-1.17.7.tgz", - "dev": true + "integrity": "sha1-RUKk9JugxFwF6y6d2dID4rjv4L8=", + "dev": true, + "requires": { + "formatio": "1.1.1", + "lolex": "1.3.2", + "samsam": "1.1.2", + "util": ">=0.10.3 <1" + } }, "socket.io": { - "version": "0.9.16", - "from": "socket.io@0.9.16", - "resolved": "https://registry.npmjs.org/socket.io/-/socket.io-0.9.16.tgz", + "version": "0.9.19", + "resolved": "https://registry.npmjs.org/socket.io/-/socket.io-0.9.19.tgz", + "integrity": "sha1-SQu1/Q3FTPAC7gTmf638Q7hIo48=", + "requires": { + "base64id": "0.1.0", + "policyfile": "0.0.4", + "redis": "0.7.3", + "socket.io-client": "0.9.16" + }, "dependencies": { "redis": { "version": "0.7.3", - "from": "redis@0.7.3", "resolved": "https://registry.npmjs.org/redis/-/redis-0.7.3.tgz", + "integrity": "sha1-7le3pE0l7BWU5ENl2BZfp9HUgRo=", "optional": true }, "socket.io-client": { "version": "0.9.16", - "from": "socket.io-client@0.9.16", - "resolved": "https://registry.npmjs.org/socket.io-client/-/socket.io-client-0.9.16.tgz" + "resolved": "https://registry.npmjs.org/socket.io-client/-/socket.io-client-0.9.16.tgz", + "integrity": "sha1-TadRXF53MEHRtCOXBBW8xDDzX8Y=", + "requires": { + "active-x-obfuscator": "0.0.1", + "uglify-js": "1.2.5", + "ws": "0.4.x", + "xmlhttprequest": "1.4.2" + } } } }, "socket.io-client": { "version": "0.9.17", - "from": "socket.io-client@>=0.9.16 <0.10.0", - "resolved": "https://registry.npmjs.org/socket.io-client/-/socket.io-client-0.9.17.tgz" + "resolved": "https://registry.npmjs.org/socket.io-client/-/socket.io-client-0.9.17.tgz", + "integrity": "sha1-MdEsRLc/u6NJY57M+DPj+a4T/m8=", + "requires": { + "active-x-obfuscator": "0.0.1", + "uglify-js": "1.2.5", + "ws": "0.4.x", + "xmlhttprequest": "1.4.2" + } }, "source-map": { "version": "0.6.1", - "from": "source-map@>=0.6.1 <0.7.0", - "resolved": "https://registry.npmjs.org/source-map/-/source-map-0.6.1.tgz" + "resolved": "https://registry.npmjs.org/source-map/-/source-map-0.6.1.tgz", + "integrity": "sha1-dHIq8y6WFOnCh6jQu95IteLxomM=" }, "split": { "version": "1.0.1", - "from": "split@>=1.0.0 <2.0.0", - "resolved": "https://registry.npmjs.org/split/-/split-1.0.1.tgz" + "resolved": "https://registry.npmjs.org/split/-/split-1.0.1.tgz", + "integrity": "sha1-YFvZvjA6pZ+zX5Ip++oN3snqB9k=", + "requires": { + "through": "2" + } }, "sshpk": { "version": "1.16.1", - "from": "sshpk@>=1.7.0 <2.0.0", "resolved": "https://registry.npmjs.org/sshpk/-/sshpk-1.16.1.tgz", + "integrity": "sha1-+2YcC+8ps520B2nuOfpwCT1vaHc=", + "requires": { + "asn1": "~0.2.3", + "assert-plus": "^1.0.0", + "bcrypt-pbkdf": "^1.0.0", + "dashdash": "^1.12.0", + "ecc-jsbn": "~0.1.1", + "getpass": "^0.1.1", + "jsbn": "~0.1.0", + "safer-buffer": "^2.0.2", + "tweetnacl": "~0.14.0" + }, "dependencies": { "asn1": { "version": "0.2.4", - "from": "asn1@>=0.2.3 <0.3.0", - "resolved": "https://registry.npmjs.org/asn1/-/asn1-0.2.4.tgz" + "resolved": "https://registry.npmjs.org/asn1/-/asn1-0.2.4.tgz", + "integrity": "sha1-jSR136tVO7M+d7VOWeiAu4ziMTY=", + "requires": { + "safer-buffer": "~2.1.0" + } }, "assert-plus": { "version": "1.0.0", - "from": "assert-plus@^1.0.0", - "resolved": "https://registry.npmjs.org/assert-plus/-/assert-plus-1.0.0.tgz" + "resolved": "https://registry.npmjs.org/assert-plus/-/assert-plus-1.0.0.tgz", + "integrity": "sha1-8S4PPF13sLHN2RRpQuTpbB5N1SU=" } } }, "stack-trace": { "version": "0.0.9", - "from": "stack-trace@0.0.9", - "resolved": "https://registry.npmjs.org/stack-trace/-/stack-trace-0.0.9.tgz" + "resolved": "https://registry.npmjs.org/stack-trace/-/stack-trace-0.0.9.tgz", + "integrity": "sha1-qPbq7KkGdMMz58Q5U/J1tFFRBpU=" }, "standard-as-callback": { "version": "2.0.1", - "from": "standard-as-callback@>=2.0.1 <3.0.0", - "resolved": "https://registry.npmjs.org/standard-as-callback/-/standard-as-callback-2.0.1.tgz" + "resolved": "https://registry.npmjs.org/standard-as-callback/-/standard-as-callback-2.0.1.tgz", + "integrity": "sha1-7YuyVkjhWDF1m2Ajvbh+a2CzgSY=" }, "statsd-parser": { "version": "0.0.4", - "from": "statsd-parser@>=0.0.4 <0.1.0", - "resolved": "https://registry.npmjs.org/statsd-parser/-/statsd-parser-0.0.4.tgz" + "resolved": "https://registry.npmjs.org/statsd-parser/-/statsd-parser-0.0.4.tgz", + "integrity": "sha1-y9JDlTzELv/VSLXSI4jtaJ7GOb0=" }, "statuses": { "version": "1.5.0", - "from": "statuses@>=1.4.0 <2.0.0", - "resolved": "https://registry.npmjs.org/statuses/-/statuses-1.5.0.tgz" + "resolved": "https://registry.npmjs.org/statuses/-/statuses-1.5.0.tgz", + "integrity": "sha1-Fhx9rBd2Wf2YEfQ3cfqZOBR4Yow=" }, "stream-shift": { "version": "1.0.0", - "from": "stream-shift@>=1.0.0 <2.0.0", - "resolved": "https://registry.npmjs.org/stream-shift/-/stream-shift-1.0.0.tgz" + "resolved": "https://registry.npmjs.org/stream-shift/-/stream-shift-1.0.0.tgz", + "integrity": "sha1-1cdSgl5TZ+eG944Y5EXqIjoVWVI=" }, "string_decoder": { "version": "1.1.1", - "from": "string_decoder@>=1.1.1 <1.2.0", - "resolved": "https://registry.npmjs.org/string_decoder/-/string_decoder-1.1.1.tgz" + "resolved": "https://registry.npmjs.org/string_decoder/-/string_decoder-1.1.1.tgz", + "integrity": "sha1-nPFhG6YmhdcDCunkujQUnDrwP8g=", + "requires": { + "safe-buffer": "~5.1.0" + } }, "tdigest": { "version": "0.1.1", - "from": "tdigest@>=0.1.1 <0.2.0", - "resolved": "https://registry.npmjs.org/tdigest/-/tdigest-0.1.1.tgz" + "resolved": "https://registry.npmjs.org/tdigest/-/tdigest-0.1.1.tgz", + "integrity": "sha1-Ljyyw56kSeVdHmzZEReszKRYgCE=", + "requires": { + "bintrees": "1.0.1" + } }, "teeny-request": { "version": "3.11.3", - "from": "teeny-request@>=3.6.0 <4.0.0", "resolved": "https://registry.npmjs.org/teeny-request/-/teeny-request-3.11.3.tgz", + "integrity": "sha1-M1xin3ZF5dZZk2LfLzIwxMvCOlU=", + "requires": { + "https-proxy-agent": "^2.2.1", + "node-fetch": "^2.2.0", + "uuid": "^3.3.2" + }, "dependencies": { "uuid": { "version": "3.3.2", - "from": "uuid@>=3.3.2 <4.0.0", - "resolved": "https://registry.npmjs.org/uuid/-/uuid-3.3.2.tgz" + "resolved": "https://registry.npmjs.org/uuid/-/uuid-3.3.2.tgz", + "integrity": "sha1-G0r0lV6zB3xQHCOHL8ZROBFYcTE=" } } }, "through": { "version": "2.3.8", - "from": "through@>=2.0.0 <3.0.0", - "resolved": "https://registry.npmjs.org/through/-/through-2.3.8.tgz" + "resolved": "https://registry.npmjs.org/through/-/through-2.3.8.tgz", + "integrity": "sha1-DdTJ/6q8NXlgsbckEV1+Doai4fU=" }, "through2": { "version": "2.0.5", - "from": "through2@>=2.0.3 <3.0.0", - "resolved": "https://registry.npmjs.org/through2/-/through2-2.0.5.tgz" + "resolved": "https://registry.npmjs.org/through2/-/through2-2.0.5.tgz", + "integrity": "sha1-AcHjnrMdB8t9A6lqcIIyYLIxMs0=", + "requires": { + "readable-stream": "~2.3.6", + "xtend": "~4.0.1" + } }, "timekeeper": { "version": "0.0.4", - "from": "timekeeper@0.0.4", "resolved": "https://registry.npmjs.org/timekeeper/-/timekeeper-0.0.4.tgz", + "integrity": "sha1-kNt58X2Ni1NiFUOJSSuXJ2LP0nY=", "dev": true }, "tinycolor": { "version": "0.0.1", - "from": "tinycolor@>=0.0.0 <1.0.0", - "resolved": "https://registry.npmjs.org/tinycolor/-/tinycolor-0.0.1.tgz" + "resolved": "https://registry.npmjs.org/tinycolor/-/tinycolor-0.0.1.tgz", + "integrity": "sha1-MgtaUtg6u1l42Bo+iH1K77FaYWQ=" }, "tough-cookie": { "version": "2.4.3", - "from": "tough-cookie@>=2.4.3 <2.5.0", - "resolved": "https://registry.npmjs.org/tough-cookie/-/tough-cookie-2.4.3.tgz" + "resolved": "https://registry.npmjs.org/tough-cookie/-/tough-cookie-2.4.3.tgz", + "integrity": "sha1-U/Nto/R3g7CSWvoG/587FlKA94E=", + "requires": { + "psl": "^1.1.24", + "punycode": "^1.4.1" + } }, "tunnel-agent": { "version": "0.6.0", - "from": "tunnel-agent@>=0.6.0 <0.7.0", - "resolved": "https://registry.npmjs.org/tunnel-agent/-/tunnel-agent-0.6.0.tgz" + "resolved": "https://registry.npmjs.org/tunnel-agent/-/tunnel-agent-0.6.0.tgz", + "integrity": "sha1-J6XeoGs2sEoKmWZ3SykIaPD8QP0=", + "requires": { + "safe-buffer": "^5.0.1" + } }, "tweetnacl": { "version": "0.14.5", - "from": "tweetnacl@>=0.14.0 <0.15.0", - "resolved": "https://registry.npmjs.org/tweetnacl/-/tweetnacl-0.14.5.tgz" + "resolved": "https://registry.npmjs.org/tweetnacl/-/tweetnacl-0.14.5.tgz", + "integrity": "sha1-WuaBd/GS1EViadEIr6k/+HQ/T2Q=" }, "type-detect": { "version": "0.1.1", - "from": "type-detect@0.1.1", "resolved": "https://registry.npmjs.org/type-detect/-/type-detect-0.1.1.tgz", + "integrity": "sha1-C6XsKohWQORw6k6FBZcZANrFiCI=", "dev": true }, "type-is": { "version": "1.6.16", - "from": "type-is@>=1.6.16 <1.7.0", - "resolved": "https://registry.npmjs.org/type-is/-/type-is-1.6.16.tgz" + "resolved": "https://registry.npmjs.org/type-is/-/type-is-1.6.16.tgz", + "integrity": "sha1-+JzjQVQcZysl7nrjxz3uOyvlAZQ=", + "requires": { + "media-typer": "0.3.0", + "mime-types": "~2.1.18" + } }, "uglify-js": { "version": "1.2.5", - "from": "uglify-js@1.2.5", - "resolved": "https://registry.npmjs.org/uglify-js/-/uglify-js-1.2.5.tgz" + "resolved": "https://registry.npmjs.org/uglify-js/-/uglify-js-1.2.5.tgz", + "integrity": "sha1-tULCx29477NLIAsgF3Y0Mw/3ArY=" }, "uid-safe": { "version": "1.1.0", - "from": "uid-safe@>=1.0.1 <2.0.0", "resolved": "https://registry.npmjs.org/uid-safe/-/uid-safe-1.1.0.tgz", - "dev": true + "integrity": "sha1-WNbF2r+N+9jVKDSDmAbAP9YUMjI=", + "dev": true, + "requires": { + "base64-url": "1.2.1", + "native-or-bluebird": "~1.1.2" + } }, "underscore": { "version": "1.7.0", - "from": "underscore@>=1.7.0 <1.8.0", - "resolved": "https://registry.npmjs.org/underscore/-/underscore-1.7.0.tgz" + "resolved": "https://registry.npmjs.org/underscore/-/underscore-1.7.0.tgz", + "integrity": "sha1-a7rwh3UA02vjTsqlhODbn+8DUgk=" }, "unpipe": { "version": "1.0.0", - "from": "unpipe@1.0.0", - "resolved": "https://registry.npmjs.org/unpipe/-/unpipe-1.0.0.tgz" + "resolved": "https://registry.npmjs.org/unpipe/-/unpipe-1.0.0.tgz", + "integrity": "sha1-sr9O6FFKrmFltIF4KdIbLvSZBOw=" }, "uri-js": { "version": "4.2.2", - "from": "uri-js@>=4.2.2 <5.0.0", "resolved": "https://registry.npmjs.org/uri-js/-/uri-js-4.2.2.tgz", + "integrity": "sha1-lMVA4f93KVbiKZUHwBCupsiDjrA=", + "requires": { + "punycode": "^2.1.0" + }, "dependencies": { "punycode": { "version": "2.1.1", - "from": "punycode@>=2.1.0 <3.0.0", - "resolved": "https://registry.npmjs.org/punycode/-/punycode-2.1.1.tgz" + "resolved": "https://registry.npmjs.org/punycode/-/punycode-2.1.1.tgz", + "integrity": "sha1-tYsBCsQMIsVldhbI0sLALHv0eew=" } } }, "util": { "version": "0.12.1", - "from": "util@>=0.10.3 <1.0.0", "resolved": "https://registry.npmjs.org/util/-/util-0.12.1.tgz", + "integrity": "sha1-+QjntjPnOWx2TmlN0U5xYlbOit4=", "dev": true, + "requires": { + "inherits": "^2.0.3", + "is-arguments": "^1.0.4", + "is-generator-function": "^1.0.7", + "object.entries": "^1.1.0", + "safe-buffer": "^5.1.2" + }, "dependencies": { "safe-buffer": { "version": "5.2.0", - "from": "safe-buffer@>=5.1.2 <6.0.0", "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.2.0.tgz", + "integrity": "sha1-t02uxJsRSPiMZLaNSbHoFcHy9Rk=", "dev": true } } }, "util-deprecate": { "version": "1.0.2", - "from": "util-deprecate@>=1.0.1 <1.1.0", - "resolved": "https://registry.npmjs.org/util-deprecate/-/util-deprecate-1.0.2.tgz" + "resolved": "https://registry.npmjs.org/util-deprecate/-/util-deprecate-1.0.2.tgz", + "integrity": "sha1-RQ1Nyfpw3nMnYvvS1KKJgUGaDM8=" }, "utils-merge": { "version": "1.0.1", - "from": "utils-merge@1.0.1", - "resolved": "https://registry.npmjs.org/utils-merge/-/utils-merge-1.0.1.tgz" + "resolved": "https://registry.npmjs.org/utils-merge/-/utils-merge-1.0.1.tgz", + "integrity": "sha1-n5VxD1CiZ5R7LMwSR0HBAoQn5xM=" }, "uuid": { "version": "3.0.0", - "from": "uuid@3.0.0", - "resolved": "https://registry.npmjs.org/uuid/-/uuid-3.0.0.tgz" + "resolved": "https://registry.npmjs.org/uuid/-/uuid-3.0.0.tgz", + "integrity": "sha1-Zyj8BFnEUNeWqZwxg3VpvfZy1yg=" }, "vary": { "version": "1.1.2", - "from": "vary@>=1.1.2 <1.2.0", - "resolved": "https://registry.npmjs.org/vary/-/vary-1.1.2.tgz" + "resolved": "https://registry.npmjs.org/vary/-/vary-1.1.2.tgz", + "integrity": "sha1-IpnwLG3tMNSllhsLn3RSShj2NPw=" }, "verror": { "version": "1.10.0", - "from": "verror@1.10.0", "resolved": "https://registry.npmjs.org/verror/-/verror-1.10.0.tgz", + "integrity": "sha1-OhBcoXBTr1XW4nDB+CiGguGNpAA=", + "requires": { + "assert-plus": "^1.0.0", + "core-util-is": "1.0.2", + "extsprintf": "^1.2.0" + }, "dependencies": { "assert-plus": { "version": "1.0.0", - "from": "assert-plus@^1.0.0", - "resolved": "https://registry.npmjs.org/assert-plus/-/assert-plus-1.0.0.tgz" + "resolved": "https://registry.npmjs.org/assert-plus/-/assert-plus-1.0.0.tgz", + "integrity": "sha1-8S4PPF13sLHN2RRpQuTpbB5N1SU=" } } }, "wrappy": { "version": "1.0.2", - "from": "wrappy@1", - "resolved": "https://registry.npmjs.org/wrappy/-/wrappy-1.0.2.tgz" + "resolved": "https://registry.npmjs.org/wrappy/-/wrappy-1.0.2.tgz", + "integrity": "sha1-tSQ9jz7BqjXxNkYFvA0QNuMKtp8=" }, "ws": { "version": "0.4.32", - "from": "ws@>=0.4.0 <0.5.0", "resolved": "https://registry.npmjs.org/ws/-/ws-0.4.32.tgz", + "integrity": "sha1-eHphVEFPPJntg8V3IVOyD+sM7DI=", + "requires": { + "commander": "~2.1.0", + "nan": "~1.0.0", + "options": ">=0.0.5", + "tinycolor": "0.x" + }, "dependencies": { "commander": { "version": "2.1.0", - "from": "commander@>=2.1.0 <2.2.0", - "resolved": "https://registry.npmjs.org/commander/-/commander-2.1.0.tgz" + "resolved": "https://registry.npmjs.org/commander/-/commander-2.1.0.tgz", + "integrity": "sha1-0SG7roYNmZKj1Re6lvVliOR8Z4E=" }, "nan": { "version": "1.0.0", - "from": "nan@>=1.0.0 <1.1.0", - "resolved": "https://registry.npmjs.org/nan/-/nan-1.0.0.tgz" + "resolved": "https://registry.npmjs.org/nan/-/nan-1.0.0.tgz", + "integrity": "sha1-riT4hQgY1mL8q1rPfzuVv6oszzg=" } } }, "xmlhttprequest": { "version": "1.4.2", - "from": "xmlhttprequest@1.4.2", - "resolved": "https://registry.npmjs.org/xmlhttprequest/-/xmlhttprequest-1.4.2.tgz" + "resolved": "https://registry.npmjs.org/xmlhttprequest/-/xmlhttprequest-1.4.2.tgz", + "integrity": "sha1-AUU6HZvtHo8XL2SVu/TIxCYyFQA=" }, "xtend": { "version": "4.0.1", - "from": "xtend@>=4.0.1 <4.1.0", - "resolved": "https://registry.npmjs.org/xtend/-/xtend-4.0.1.tgz" + "resolved": "https://registry.npmjs.org/xtend/-/xtend-4.0.1.tgz", + "integrity": "sha1-pcbVMr5lbiPbgg77lDofBJmNY68=" }, "yallist": { "version": "3.0.3", - "from": "yallist@>=3.0.2 <4.0.0", - "resolved": "https://registry.npmjs.org/yallist/-/yallist-3.0.3.tgz" + "resolved": "https://registry.npmjs.org/yallist/-/yallist-3.0.3.tgz", + "integrity": "sha1-tLBJ4xS+VF486AIjbWzSLNkcPek=" }, "zeparser": { "version": "0.0.5", - "from": "zeparser@0.0.5", - "resolved": "https://registry.npmjs.org/zeparser/-/zeparser-0.0.5.tgz" + "resolved": "https://registry.npmjs.org/zeparser/-/zeparser-0.0.5.tgz", + "integrity": "sha1-A3JlYbwmjy5URPVMZlt/1KjAKeI=" } } } diff --git a/services/real-time/package.json b/services/real-time/package.json index ba94412efe..e6c630b662 100644 --- a/services/real-time/package.json +++ b/services/real-time/package.json @@ -34,7 +34,7 @@ "request": "^2.88.0", "session.socket.io": "^0.1.6", "settings-sharelatex": "^1.1.0", - "socket.io": "0.9.16", + "socket.io": "0.9.19", "socket.io-client": "^0.9.16" }, "devDependencies": { From c7e2b99a7b6ebdf8fda960ecd0e392c9a9ca4603 Mon Sep 17 00:00:00 2001 From: Simon Detheridge Date: Tue, 4 Feb 2020 10:43:06 +0000 Subject: [PATCH 303/491] Update hybi-16 patch to work with socket.io 0.9.19 --- services/real-time/socket.io.patch.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/services/real-time/socket.io.patch.js b/services/real-time/socket.io.patch.js index 753f44b8ca..796f8c7574 100644 --- a/services/real-time/socket.io.patch.js +++ b/services/real-time/socket.io.patch.js @@ -6,7 +6,7 @@ if(process.versions.node.split('.')[0] >= 7) { var io = require("socket.io"); -if (io.version === "0.9.16") { +if (io.version === "0.9.16" || io.version === "0.9.19") { console.log("patching socket.io hybi-16 transport frame prototype"); var transports = require("socket.io/lib/transports/websocket/hybi-16.js"); transports.prototype.frame = patchedFrameHandler; From 1fc8cc44c339ace4b7eed6f326088c5ac2d3b7c6 Mon Sep 17 00:00:00 2001 From: Brian Gough Date: Tue, 4 Feb 2020 11:14:14 +0000 Subject: [PATCH 304/491] log shutdown messages as warnings --- services/real-time/app.coffee | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/services/real-time/app.coffee b/services/real-time/app.coffee index c2f9e7fa9c..3379d3e73b 100644 --- a/services/real-time/app.coffee +++ b/services/real-time/app.coffee @@ -111,17 +111,17 @@ Error.stackTraceLimit = 10 shutdownCleanly = (signal) -> connectedClients = io.sockets.clients()?.length if connectedClients == 0 - logger.log("no clients connected, exiting") + logger.warn("no clients connected, exiting") process.exit() else - logger.log {connectedClients}, "clients still connected, not shutting down yet" + logger.warn {connectedClients}, "clients still connected, not shutting down yet" setTimeout () -> shutdownCleanly(signal) - , 10000 + , 30 * 1000 drainAndShutdown = (signal) -> if Settings.shutDownInProgress - logger.log signal: signal, "shutdown already in progress, ignoring signal" + logger.warn signal: signal, "shutdown already in progress, ignoring signal" return else Settings.shutDownInProgress = true From e263d3747665f8dac2cde9bb10e9856b6d67c77c Mon Sep 17 00:00:00 2001 From: Brian Gough Date: Tue, 4 Feb 2020 11:14:53 +0000 Subject: [PATCH 305/491] pass the signal correctly to the shutdown handler --- services/real-time/app.coffee | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/services/real-time/app.coffee b/services/real-time/app.coffee index 3379d3e73b..074d384665 100644 --- a/services/real-time/app.coffee +++ b/services/real-time/app.coffee @@ -135,8 +135,7 @@ if Settings.shutdownDrainTimeWindow? shutdownDrainTimeWindow = parseInt(Settings.shutdownDrainTimeWindow, 10) logger.log shutdownDrainTimeWindow: shutdownDrainTimeWindow,"shutdownDrainTimeWindow enabled" for signal in ['SIGINT', 'SIGHUP', 'SIGQUIT', 'SIGUSR1', 'SIGUSR2', 'SIGTERM', 'SIGABRT'] - process.on signal, -> - drainAndShutdown(signal) + process.on signal, drainAndShutdown # signal is passed as argument to event handler # global exception handler if Settings.errors?.catchUncaughtErrors From 7380d523d5f57a616cb66e7ac05dcbddc71f0620 Mon Sep 17 00:00:00 2001 From: Brian Gough Date: Tue, 4 Feb 2020 11:39:37 +0000 Subject: [PATCH 306/491] avoid emitting when client not connected the emit is happening asynchronously after the client list is computed, so clients may have disconnected in the intervening time. --- services/real-time/app/coffee/WebsocketLoadBalancer.coffee | 3 +++ 1 file changed, 3 insertions(+) diff --git a/services/real-time/app/coffee/WebsocketLoadBalancer.coffee b/services/real-time/app/coffee/WebsocketLoadBalancer.coffee index 12b7ef812b..a93f5f9c9c 100644 --- a/services/real-time/app/coffee/WebsocketLoadBalancer.coffee +++ b/services/real-time/app/coffee/WebsocketLoadBalancer.coffee @@ -96,6 +96,9 @@ module.exports = WebsocketLoadBalancer = , (client, cb) -> Utils.getClientAttributes client, ['is_restricted_user'], (err, {is_restricted_user}) -> return cb(err) if err? + if !client.connected + logger.warn {channel:channel, client: client.id}, "skipping emit, client not connected" + return cb() if !seen[client.id] seen[client.id] = true if !(is_restricted_user && message.message not in RESTRICTED_USER_MESSAGE_TYPE_PASS_LIST) From ebb83e463376470a857e0a1fbf704ac5227e30de Mon Sep 17 00:00:00 2001 From: Brian Gough Date: Tue, 4 Feb 2020 11:58:55 +0000 Subject: [PATCH 307/491] use diconnected property, not connected --- services/real-time/app/coffee/WebsocketLoadBalancer.coffee | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/services/real-time/app/coffee/WebsocketLoadBalancer.coffee b/services/real-time/app/coffee/WebsocketLoadBalancer.coffee index a93f5f9c9c..e9a72dbc6a 100644 --- a/services/real-time/app/coffee/WebsocketLoadBalancer.coffee +++ b/services/real-time/app/coffee/WebsocketLoadBalancer.coffee @@ -96,7 +96,7 @@ module.exports = WebsocketLoadBalancer = , (client, cb) -> Utils.getClientAttributes client, ['is_restricted_user'], (err, {is_restricted_user}) -> return cb(err) if err? - if !client.connected + if client.disconnected logger.warn {channel:channel, client: client.id}, "skipping emit, client not connected" return cb() if !seen[client.id] From 216a97792271367d01d76610b44ce8cd4646b0af Mon Sep 17 00:00:00 2001 From: Simon Detheridge Date: Tue, 4 Feb 2020 12:13:03 +0000 Subject: [PATCH 308/491] Add try/catch around all client emissions --- .../coffee/DocumentUpdaterController.coffee | 17 +++++++--- .../real-time/app/coffee/DrainManager.coffee | 5 ++- services/real-time/app/coffee/Router.coffee | 34 ++++++++++++++----- .../app/coffee/WebsocketLoadBalancer.coffee | 10 ++++-- 4 files changed, 50 insertions(+), 16 deletions(-) diff --git a/services/real-time/app/coffee/DocumentUpdaterController.coffee b/services/real-time/app/coffee/DocumentUpdaterController.coffee index 2611d484ad..c96282f49e 100644 --- a/services/real-time/app/coffee/DocumentUpdaterController.coffee +++ b/services/real-time/app/coffee/DocumentUpdaterController.coffee @@ -71,10 +71,16 @@ module.exports = DocumentUpdaterController = seen[client.id] = true if client.id == update.meta.source logger.log doc_id: doc_id, version: update.v, source: update.meta?.source, "distributing update to sender" - client.emit "otUpdateApplied", v: update.v, doc: update.doc + try + client.emit "otUpdateApplied", v: update.v, doc: update.doc + catch err + logger.warn client_id: client.id, doc_id: doc_id, err: err, "error sending update to sender" else if !update.dup # Duplicate ops should just be sent back to sending client for acknowledgement logger.log doc_id: doc_id, version: update.v, source: update.meta?.source, client_id: client.id, "distributing update to collaborator" - client.emit "otUpdateApplied", update + try + client.emit "otUpdateApplied", update + catch err + logger.warn client_id: client.id, doc_id: doc_id, err: err, "error sending update to collaborator" if Object.keys(seen).length < clientList.length metrics.inc "socket-io.duplicate-clients", 0.1 logger.log doc_id: doc_id, socketIoClients: (client.id for client in clientList), "discarded duplicate clients" @@ -82,7 +88,10 @@ module.exports = DocumentUpdaterController = _processErrorFromDocumentUpdater: (io, doc_id, error, message) -> for client in io.sockets.clients(doc_id) logger.warn err: error, doc_id: doc_id, client_id: client.id, "error from document updater, disconnecting client" - client.emit "otUpdateError", error, message - client.disconnect() + try + client.emit "otUpdateError", error, message + client.disconnect() + catch err + logger.warn client_id: client.id, doc_id: doc_id, err: err, cause: error, "error sending error to client" diff --git a/services/real-time/app/coffee/DrainManager.coffee b/services/real-time/app/coffee/DrainManager.coffee index 2590a96726..1a8fb73be1 100644 --- a/services/real-time/app/coffee/DrainManager.coffee +++ b/services/real-time/app/coffee/DrainManager.coffee @@ -30,7 +30,10 @@ module.exports = DrainManager = if !@RECONNECTED_CLIENTS[client.id] @RECONNECTED_CLIENTS[client.id] = true logger.log {client_id: client.id}, "Asking client to reconnect gracefully" - client.emit "reconnectGracefully" + try + client.emit "reconnectGracefully" + catch err + logger.warn client_id: client.id, err: err, "error asking client to reconnect gracefully" drainedCount++ haveDrainedNClients = (drainedCount == N) if haveDrainedNClients diff --git a/services/real-time/app/coffee/Router.coffee b/services/real-time/app/coffee/Router.coffee index fcdb0c93dc..d5a7472ab0 100644 --- a/services/real-time/app/coffee/Router.coffee +++ b/services/real-time/app/coffee/Router.coffee @@ -45,29 +45,45 @@ module.exports = Router = client?.on "error", (err) -> logger.err { clientErr: err }, "socket.io client error" if client.connected - client.emit("reconnectGracefully") - client.disconnect() + try + client.emit("reconnectGracefully") + client.disconnect() + catch error + logger.warn error: error, cause: err, client_id: client.id, "error telling client to reconnect after error" if settings.shutDownInProgress - client.emit("connectionRejected", {message: "retry"}) - client.disconnect() + try + client.emit("connectionRejected", {message: "retry"}) + client.disconnect() + catch error + logger.warn error: error, client_id: client.id, message: "retry", "error rejecting client connection" return if client? and error?.message?.match(/could not look up session by key/) logger.warn err: error, client: client?, session: session?, "invalid session" # tell the client to reauthenticate if it has an invalid session key - client.emit("connectionRejected", {message: "invalid session"}) - client.disconnect() + try + client.emit("connectionRejected", {message: "invalid session"}) + client.disconnect() + catch error + logger.warn error: error, client_id: client.id, message: "invalid session", "error rejecting client connection" return if error? logger.err err: error, client: client?, session: session?, "error when client connected" - client?.emit("connectionRejected", {message: "error"}) - client?.disconnect() + try + client?.emit("connectionRejected", {message: "error"}) + client?.disconnect() + catch error + logger.warn error: error, client_id: client?.id, message: "error", "error rejecting client connection" return # send positive confirmation that the client has a valid connection - client.emit("connectionAccepted") + try + client.emit("connectionAccepted") + catch error + logger.warn error: error, client_id: client.id, "error accepting client connection" + return metrics.inc('socket-io.connection') metrics.gauge('socket-io.clients', io.sockets.clients()?.length) diff --git a/services/real-time/app/coffee/WebsocketLoadBalancer.coffee b/services/real-time/app/coffee/WebsocketLoadBalancer.coffee index 12b7ef812b..df81a04395 100644 --- a/services/real-time/app/coffee/WebsocketLoadBalancer.coffee +++ b/services/real-time/app/coffee/WebsocketLoadBalancer.coffee @@ -67,7 +67,10 @@ module.exports = WebsocketLoadBalancer = logger.error {err: error, channel}, "error parsing JSON" return if message.room_id == "all" - io.sockets.emit(message.message, message.payload...) + try + io.sockets.emit(message.message, message.payload...) + catch error + logger.warn message: message, error: error, "error sending message to clients" else if message.message is 'clientTracking.refresh' && message.room_id? clientList = io.sockets.clients(message.room_id) logger.log {channel:channel, message: message.message, room_id: message.room_id, message_id: message._id, socketIoClients: (client.id for client in clientList)}, "refreshing client list" @@ -99,7 +102,10 @@ module.exports = WebsocketLoadBalancer = if !seen[client.id] seen[client.id] = true if !(is_restricted_user && message.message not in RESTRICTED_USER_MESSAGE_TYPE_PASS_LIST) - client.emit(message.message, message.payload...) + try + client.emit(message.message, message.payload...) + catch error + console.log(message: message, client_id: client.id, error: error, "error sending message to client") cb() , (err) -> if err? From fbff3fe72720c79d79ab99999704112159d26a5d Mon Sep 17 00:00:00 2001 From: Simon Detheridge Date: Tue, 4 Feb 2020 12:56:43 +0000 Subject: [PATCH 309/491] Don't shut down on uncaught EPIPE --- services/real-time/app.coffee | 2 ++ 1 file changed, 2 insertions(+) diff --git a/services/real-time/app.coffee b/services/real-time/app.coffee index 074d384665..1aeaec4718 100644 --- a/services/real-time/app.coffee +++ b/services/real-time/app.coffee @@ -141,6 +141,8 @@ if Settings.shutdownDrainTimeWindow? if Settings.errors?.catchUncaughtErrors process.removeAllListeners('uncaughtException') process.on 'uncaughtException', (error) -> + if error.code == 'EPIPE' + return logger.warn err: error, 'attempted to write to disconnected client' logger.error err: error, 'uncaught exception' if Settings.errors?.shutdownOnUncaughtError drainAndShutdown('SIGABRT') From 8e45a62e32260f095c4622d462ca896e7b6c9771 Mon Sep 17 00:00:00 2001 From: Simon Detheridge Date: Tue, 4 Feb 2020 13:58:45 +0000 Subject: [PATCH 310/491] Handle ECONNRESET in the same way as EPIPE --- services/real-time/app.coffee | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/services/real-time/app.coffee b/services/real-time/app.coffee index 1aeaec4718..b27e2dd0f9 100644 --- a/services/real-time/app.coffee +++ b/services/real-time/app.coffee @@ -141,7 +141,8 @@ if Settings.shutdownDrainTimeWindow? if Settings.errors?.catchUncaughtErrors process.removeAllListeners('uncaughtException') process.on 'uncaughtException', (error) -> - if error.code == 'EPIPE' + if ['EPIPE', 'ECONNRESET'].includes(error.code) + Metrics.inc('disconnected_write') return logger.warn err: error, 'attempted to write to disconnected client' logger.error err: error, 'uncaught exception' if Settings.errors?.shutdownOnUncaughtError From 4102aa0580808ac08b6d63c6571a922c20604b11 Mon Sep 17 00:00:00 2001 From: Simon Detheridge Date: Tue, 4 Feb 2020 14:03:56 +0000 Subject: [PATCH 311/491] Add more detail to metric --- services/real-time/app.coffee | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/services/real-time/app.coffee b/services/real-time/app.coffee index b27e2dd0f9..677737f862 100644 --- a/services/real-time/app.coffee +++ b/services/real-time/app.coffee @@ -142,7 +142,7 @@ if Settings.shutdownDrainTimeWindow? process.removeAllListeners('uncaughtException') process.on 'uncaughtException', (error) -> if ['EPIPE', 'ECONNRESET'].includes(error.code) - Metrics.inc('disconnected_write') + Metrics.inc('disconnected_write', 1, {status: error.code}) return logger.warn err: error, 'attempted to write to disconnected client' logger.error err: error, 'uncaught exception' if Settings.errors?.shutdownOnUncaughtError From 4ec82b1baae81c41d66c8ec0a341d7e325212288 Mon Sep 17 00:00:00 2001 From: Brian Gough Date: Tue, 4 Feb 2020 15:02:15 +0000 Subject: [PATCH 312/491] upgrade to local node:10.18.1 image --- services/real-time/Dockerfile | 4 ++-- services/real-time/docker-compose.yml | 6 +----- 2 files changed, 3 insertions(+), 7 deletions(-) diff --git a/services/real-time/Dockerfile b/services/real-time/Dockerfile index 7656cf4526..5d9b04361a 100644 --- a/services/real-time/Dockerfile +++ b/services/real-time/Dockerfile @@ -1,4 +1,4 @@ -FROM node:10.16.3 as app +FROM gcr.io/overleaf-ops/node:10.18.1 as app WORKDIR /app @@ -12,7 +12,7 @@ COPY . /app RUN npm run compile:all -FROM node:10.16.3 +FROM gcr.io/overleaf-ops/node:10.18.1 COPY --from=app /app /app diff --git a/services/real-time/docker-compose.yml b/services/real-time/docker-compose.yml index 172de89b22..91a92b0055 100644 --- a/services/real-time/docker-compose.yml +++ b/services/real-time/docker-compose.yml @@ -7,7 +7,7 @@ version: "2" services: test_unit: - image: node:6.15.1 + image: gcr.io/overleaf-ops/node:10.18.1 volumes: - .:/app working_dir: /app @@ -36,8 +36,6 @@ services: - redis command: npm run test:acceptance - - tar: build: . image: ci/$PROJECT_NAME:$BRANCH_NAME-$BUILD_NUMBER @@ -51,5 +49,3 @@ services: mongo: image: mongo:3.4 - - From 64bd739a872006bc84099474fa6dfa1ddbbac30d Mon Sep 17 00:00:00 2001 From: Brian Gough Date: Wed, 5 Feb 2020 10:05:36 +0000 Subject: [PATCH 313/491] Revert "Merge pull request #91 from overleaf/spd-trycatch-all-the-things" This reverts commit 2bf7f14f9d050c58f141f465633bb6e274b903dd, reversing changes made to 989240812532ca43a52513339f4dda8f44a80a64. --- .../coffee/DocumentUpdaterController.coffee | 17 +++------- .../real-time/app/coffee/DrainManager.coffee | 5 +-- services/real-time/app/coffee/Router.coffee | 34 +++++-------------- .../app/coffee/WebsocketLoadBalancer.coffee | 13 ++----- 4 files changed, 16 insertions(+), 53 deletions(-) diff --git a/services/real-time/app/coffee/DocumentUpdaterController.coffee b/services/real-time/app/coffee/DocumentUpdaterController.coffee index c96282f49e..2611d484ad 100644 --- a/services/real-time/app/coffee/DocumentUpdaterController.coffee +++ b/services/real-time/app/coffee/DocumentUpdaterController.coffee @@ -71,16 +71,10 @@ module.exports = DocumentUpdaterController = seen[client.id] = true if client.id == update.meta.source logger.log doc_id: doc_id, version: update.v, source: update.meta?.source, "distributing update to sender" - try - client.emit "otUpdateApplied", v: update.v, doc: update.doc - catch err - logger.warn client_id: client.id, doc_id: doc_id, err: err, "error sending update to sender" + client.emit "otUpdateApplied", v: update.v, doc: update.doc else if !update.dup # Duplicate ops should just be sent back to sending client for acknowledgement logger.log doc_id: doc_id, version: update.v, source: update.meta?.source, client_id: client.id, "distributing update to collaborator" - try - client.emit "otUpdateApplied", update - catch err - logger.warn client_id: client.id, doc_id: doc_id, err: err, "error sending update to collaborator" + client.emit "otUpdateApplied", update if Object.keys(seen).length < clientList.length metrics.inc "socket-io.duplicate-clients", 0.1 logger.log doc_id: doc_id, socketIoClients: (client.id for client in clientList), "discarded duplicate clients" @@ -88,10 +82,7 @@ module.exports = DocumentUpdaterController = _processErrorFromDocumentUpdater: (io, doc_id, error, message) -> for client in io.sockets.clients(doc_id) logger.warn err: error, doc_id: doc_id, client_id: client.id, "error from document updater, disconnecting client" - try - client.emit "otUpdateError", error, message - client.disconnect() - catch err - logger.warn client_id: client.id, doc_id: doc_id, err: err, cause: error, "error sending error to client" + client.emit "otUpdateError", error, message + client.disconnect() diff --git a/services/real-time/app/coffee/DrainManager.coffee b/services/real-time/app/coffee/DrainManager.coffee index 1a8fb73be1..2590a96726 100644 --- a/services/real-time/app/coffee/DrainManager.coffee +++ b/services/real-time/app/coffee/DrainManager.coffee @@ -30,10 +30,7 @@ module.exports = DrainManager = if !@RECONNECTED_CLIENTS[client.id] @RECONNECTED_CLIENTS[client.id] = true logger.log {client_id: client.id}, "Asking client to reconnect gracefully" - try - client.emit "reconnectGracefully" - catch err - logger.warn client_id: client.id, err: err, "error asking client to reconnect gracefully" + client.emit "reconnectGracefully" drainedCount++ haveDrainedNClients = (drainedCount == N) if haveDrainedNClients diff --git a/services/real-time/app/coffee/Router.coffee b/services/real-time/app/coffee/Router.coffee index d5a7472ab0..fcdb0c93dc 100644 --- a/services/real-time/app/coffee/Router.coffee +++ b/services/real-time/app/coffee/Router.coffee @@ -45,45 +45,29 @@ module.exports = Router = client?.on "error", (err) -> logger.err { clientErr: err }, "socket.io client error" if client.connected - try - client.emit("reconnectGracefully") - client.disconnect() - catch error - logger.warn error: error, cause: err, client_id: client.id, "error telling client to reconnect after error" + client.emit("reconnectGracefully") + client.disconnect() if settings.shutDownInProgress - try - client.emit("connectionRejected", {message: "retry"}) - client.disconnect() - catch error - logger.warn error: error, client_id: client.id, message: "retry", "error rejecting client connection" + client.emit("connectionRejected", {message: "retry"}) + client.disconnect() return if client? and error?.message?.match(/could not look up session by key/) logger.warn err: error, client: client?, session: session?, "invalid session" # tell the client to reauthenticate if it has an invalid session key - try - client.emit("connectionRejected", {message: "invalid session"}) - client.disconnect() - catch error - logger.warn error: error, client_id: client.id, message: "invalid session", "error rejecting client connection" + client.emit("connectionRejected", {message: "invalid session"}) + client.disconnect() return if error? logger.err err: error, client: client?, session: session?, "error when client connected" - try - client?.emit("connectionRejected", {message: "error"}) - client?.disconnect() - catch error - logger.warn error: error, client_id: client?.id, message: "error", "error rejecting client connection" + client?.emit("connectionRejected", {message: "error"}) + client?.disconnect() return # send positive confirmation that the client has a valid connection - try - client.emit("connectionAccepted") - catch error - logger.warn error: error, client_id: client.id, "error accepting client connection" - return + client.emit("connectionAccepted") metrics.inc('socket-io.connection') metrics.gauge('socket-io.clients', io.sockets.clients()?.length) diff --git a/services/real-time/app/coffee/WebsocketLoadBalancer.coffee b/services/real-time/app/coffee/WebsocketLoadBalancer.coffee index 64274cc2bd..12b7ef812b 100644 --- a/services/real-time/app/coffee/WebsocketLoadBalancer.coffee +++ b/services/real-time/app/coffee/WebsocketLoadBalancer.coffee @@ -67,10 +67,7 @@ module.exports = WebsocketLoadBalancer = logger.error {err: error, channel}, "error parsing JSON" return if message.room_id == "all" - try - io.sockets.emit(message.message, message.payload...) - catch error - logger.warn message: message, error: error, "error sending message to clients" + io.sockets.emit(message.message, message.payload...) else if message.message is 'clientTracking.refresh' && message.room_id? clientList = io.sockets.clients(message.room_id) logger.log {channel:channel, message: message.message, room_id: message.room_id, message_id: message._id, socketIoClients: (client.id for client in clientList)}, "refreshing client list" @@ -99,16 +96,10 @@ module.exports = WebsocketLoadBalancer = , (client, cb) -> Utils.getClientAttributes client, ['is_restricted_user'], (err, {is_restricted_user}) -> return cb(err) if err? - if client.disconnected - logger.warn {channel:channel, client: client.id}, "skipping emit, client not connected" - return cb() if !seen[client.id] seen[client.id] = true if !(is_restricted_user && message.message not in RESTRICTED_USER_MESSAGE_TYPE_PASS_LIST) - try - client.emit(message.message, message.payload...) - catch error - console.log(message: message, client_id: client.id, error: error, "error sending message to client") + client.emit(message.message, message.payload...) cb() , (err) -> if err? From abe4d1d52516115d69dd3ba2ad81e79bdad453df Mon Sep 17 00:00:00 2001 From: Brian Gough Date: Thu, 6 Feb 2020 03:34:30 +0000 Subject: [PATCH 314/491] update to gcr.io/overleaf-ops/node:10.19.0 --- services/real-time/Dockerfile | 4 ++-- services/real-time/docker-compose.yml | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/services/real-time/Dockerfile b/services/real-time/Dockerfile index 5d9b04361a..cbe222e706 100644 --- a/services/real-time/Dockerfile +++ b/services/real-time/Dockerfile @@ -1,4 +1,4 @@ -FROM gcr.io/overleaf-ops/node:10.18.1 as app +FROM gcr.io/overleaf-ops/node:10.19.0 as app WORKDIR /app @@ -12,7 +12,7 @@ COPY . /app RUN npm run compile:all -FROM gcr.io/overleaf-ops/node:10.18.1 +FROM gcr.io/overleaf-ops/node:10.19.0 COPY --from=app /app /app diff --git a/services/real-time/docker-compose.yml b/services/real-time/docker-compose.yml index 91a92b0055..da376acc8c 100644 --- a/services/real-time/docker-compose.yml +++ b/services/real-time/docker-compose.yml @@ -7,7 +7,7 @@ version: "2" services: test_unit: - image: gcr.io/overleaf-ops/node:10.18.1 + image: gcr.io/overleaf-ops/node:10.19.0 volumes: - .:/app working_dir: /app From 04a9d66784ffcc8e2530bd704f6c9a6fab6372e7 Mon Sep 17 00:00:00 2001 From: Brian Gough Date: Fri, 7 Feb 2020 14:15:48 +0000 Subject: [PATCH 315/491] use public node:10.19.0 image --- services/real-time/Dockerfile | 4 ++-- services/real-time/docker-compose.yml | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/services/real-time/Dockerfile b/services/real-time/Dockerfile index cbe222e706..dee218ce8c 100644 --- a/services/real-time/Dockerfile +++ b/services/real-time/Dockerfile @@ -1,4 +1,4 @@ -FROM gcr.io/overleaf-ops/node:10.19.0 as app +FROM node:10.19.0 as app WORKDIR /app @@ -12,7 +12,7 @@ COPY . /app RUN npm run compile:all -FROM gcr.io/overleaf-ops/node:10.19.0 +FROM node:10.19.0 COPY --from=app /app /app diff --git a/services/real-time/docker-compose.yml b/services/real-time/docker-compose.yml index da376acc8c..f80231bd19 100644 --- a/services/real-time/docker-compose.yml +++ b/services/real-time/docker-compose.yml @@ -7,7 +7,7 @@ version: "2" services: test_unit: - image: gcr.io/overleaf-ops/node:10.19.0 + image: node:10.19.0 volumes: - .:/app working_dir: /app From e0e2090a422c5d6136a2ca3177efc04446261fd1 Mon Sep 17 00:00:00 2001 From: Brian Gough Date: Fri, 7 Feb 2020 14:41:12 +0000 Subject: [PATCH 316/491] update node version in nvmrc and buildscripts --- services/real-time/.nvmrc | 2 +- services/real-time/buildscript.txt | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/services/real-time/.nvmrc b/services/real-time/.nvmrc index d36e8d82f3..5b7269c0a9 100644 --- a/services/real-time/.nvmrc +++ b/services/real-time/.nvmrc @@ -1 +1 @@ -6.15.1 +10.19.0 diff --git a/services/real-time/buildscript.txt b/services/real-time/buildscript.txt index 1818ad74a9..712fc87174 100644 --- a/services/real-time/buildscript.txt +++ b/services/real-time/buildscript.txt @@ -1,6 +1,6 @@ real-time --language=coffeescript ---node-version=6.15.1 +--node-version=10.19.0 --acceptance-creds=None --dependencies=mongo,redis --docker-repos=gcr.io/overleaf-ops From 24d46e9d4bb8e9c7689fd3fefb432a28041800a8 Mon Sep 17 00:00:00 2001 From: Jakob Ackermann Date: Mon, 10 Feb 2020 17:10:41 +0100 Subject: [PATCH 317/491] [misc] update the build scripts to 1.3.5 --- services/real-time/Dockerfile | 13 +++++++++---- services/real-time/Jenkinsfile | 1 + services/real-time/Makefile | 9 ++++++++- services/real-time/buildscript.txt | 6 +++--- services/real-time/docker-compose.ci.yml | 14 +++++++------- services/real-time/docker-compose.yml | 23 +++++++++-------------- 6 files changed, 37 insertions(+), 29 deletions(-) diff --git a/services/real-time/Dockerfile b/services/real-time/Dockerfile index dee218ce8c..e538fb48d9 100644 --- a/services/real-time/Dockerfile +++ b/services/real-time/Dockerfile @@ -1,7 +1,14 @@ -FROM node:10.19.0 as app +# This file was auto-generated, do not edit it directly. +# Instead run bin/update_build_scripts from +# https://github.com/sharelatex/sharelatex-dev-environment +# Version: 1.3.5 + +FROM node:10.19.0 as base WORKDIR /app +FROM base as app + #wildcard as some files may not be in all repos COPY package*.json npm-shrink*.json /app/ @@ -12,11 +19,9 @@ COPY . /app RUN npm run compile:all -FROM node:10.19.0 +FROM base COPY --from=app /app /app - -WORKDIR /app USER node CMD ["node", "--expose-gc", "app.js"] diff --git a/services/real-time/Jenkinsfile b/services/real-time/Jenkinsfile index 792a7a6afa..684fb7daca 100644 --- a/services/real-time/Jenkinsfile +++ b/services/real-time/Jenkinsfile @@ -16,6 +16,7 @@ pipeline { } stages { + stage('Install') { steps { withCredentials([usernamePassword(credentialsId: 'GITHUB_INTEGRATION', usernameVariable: 'GH_AUTH_USERNAME', passwordVariable: 'GH_AUTH_PASSWORD')]) { diff --git a/services/real-time/Makefile b/services/real-time/Makefile index 487735cf19..5dee366537 100644 --- a/services/real-time/Makefile +++ b/services/real-time/Makefile @@ -1,7 +1,7 @@ # This file was auto-generated, do not edit it directly. # Instead run bin/update_build_scripts from # https://github.com/sharelatex/sharelatex-dev-environment -# Version: 1.1.24 +# Version: 1.3.5 BUILD_NUMBER ?= local BRANCH_NAME ?= $(shell git rev-parse --abbrev-ref HEAD) @@ -28,14 +28,20 @@ test_unit: test_acceptance: test_clean test_acceptance_pre_run test_acceptance_run +test_acceptance_debug: test_clean test_acceptance_pre_run test_acceptance_run_debug + test_acceptance_run: @[ ! -d test/acceptance ] && echo "real-time has no acceptance tests" || $(DOCKER_COMPOSE) run --rm test_acceptance +test_acceptance_run_debug: + @[ ! -d test/acceptance ] && echo "real-time has no acceptance tests" || $(DOCKER_COMPOSE) run -p 127.0.0.9:19999:19999 --rm test_acceptance npm run test:acceptance -- --inspect=0.0.0.0:19999 --inspect-brk + test_clean: $(DOCKER_COMPOSE) down -v -t 0 test_acceptance_pre_run: @[ ! -f test/acceptance/js/scripts/pre-run ] && echo "real-time has no pre acceptance tests task" || $(DOCKER_COMPOSE) run --rm test_acceptance test/acceptance/js/scripts/pre-run + build: docker build --pull --tag ci/$(PROJECT_NAME):$(BRANCH_NAME)-$(BUILD_NUMBER) \ --tag gcr.io/overleaf-ops/$(PROJECT_NAME):$(BRANCH_NAME)-$(BUILD_NUMBER) \ @@ -48,4 +54,5 @@ publish: docker push $(DOCKER_REPO)/$(PROJECT_NAME):$(BRANCH_NAME)-$(BUILD_NUMBER) + .PHONY: clean test test_unit test_acceptance test_clean build publish diff --git a/services/real-time/buildscript.txt b/services/real-time/buildscript.txt index 712fc87174..8d1c80f036 100644 --- a/services/real-time/buildscript.txt +++ b/services/real-time/buildscript.txt @@ -1,10 +1,10 @@ real-time +--public-repo=True --language=coffeescript +--env-add= --node-version=10.19.0 --acceptance-creds=None --dependencies=mongo,redis --docker-repos=gcr.io/overleaf-ops ---build-target=docker ---script-version=1.1.24 --env-pass-through= ---public-repo=True +--script-version=1.3.5 diff --git a/services/real-time/docker-compose.ci.yml b/services/real-time/docker-compose.ci.yml index c78d90e8ed..b99da9b18e 100644 --- a/services/real-time/docker-compose.ci.yml +++ b/services/real-time/docker-compose.ci.yml @@ -1,9 +1,9 @@ # This file was auto-generated, do not edit it directly. # Instead run bin/update_build_scripts from # https://github.com/sharelatex/sharelatex-dev-environment -# Version: 1.1.24 +# Version: 1.3.5 -version: "2" +version: "2.3" services: test_unit: @@ -25,13 +25,14 @@ services: MOCHA_GREP: ${MOCHA_GREP} NODE_ENV: test depends_on: - - mongo - - redis + mongo: + condition: service_healthy + redis: + condition: service_healthy user: node command: npm run test:acceptance:_run - tar: build: . image: ci/$PROJECT_NAME:$BRANCH_NAME-$BUILD_NUMBER @@ -39,9 +40,8 @@ services: - ./:/tmp/build/ command: tar -czf /tmp/build/build.tar.gz --exclude=build.tar.gz --exclude-vcs . user: root - redis: image: redis mongo: - image: mongo:3.4 + image: mongo:3.6 diff --git a/services/real-time/docker-compose.yml b/services/real-time/docker-compose.yml index f80231bd19..6a1bbb1005 100644 --- a/services/real-time/docker-compose.yml +++ b/services/real-time/docker-compose.yml @@ -1,9 +1,9 @@ # This file was auto-generated, do not edit it directly. # Instead run bin/update_build_scripts from # https://github.com/sharelatex/sharelatex-dev-environment -# Version: 1.1.24 +# Version: 1.3.5 -version: "2" +version: "2.3" services: test_unit: @@ -18,7 +18,7 @@ services: user: node test_acceptance: - build: . + image: node:10.19.0 volumes: - .:/app working_dir: /app @@ -32,20 +32,15 @@ services: NODE_ENV: test user: node depends_on: - - mongo - - redis + mongo: + condition: service_healthy + redis: + condition: service_healthy command: npm run test:acceptance - tar: - build: . - image: ci/$PROJECT_NAME:$BRANCH_NAME-$BUILD_NUMBER - volumes: - - ./:/tmp/build/ - command: tar -czf /tmp/build/build.tar.gz --exclude=build.tar.gz --exclude-vcs . - user: root - redis: image: redis mongo: - image: mongo:3.4 + image: mongo:3.6 + From 1ad8315437ece7f7139e6367df360ec5cc6cb25d Mon Sep 17 00:00:00 2001 From: Brian Gough Date: Wed, 12 Feb 2020 12:37:00 +0000 Subject: [PATCH 318/491] remove unused .travis.yml file --- services/real-time/.travis.yml | 12 ------------ 1 file changed, 12 deletions(-) delete mode 100644 services/real-time/.travis.yml diff --git a/services/real-time/.travis.yml b/services/real-time/.travis.yml deleted file mode 100644 index 311f2fd9dd..0000000000 --- a/services/real-time/.travis.yml +++ /dev/null @@ -1,12 +0,0 @@ -language: node_js - -before_install: - - npm install -g grunt-cli - -install: - - npm install - - grunt install - -script: - - grunt test:unit - - grunt compile:acceptance_tests From 902b4fca46a90a670b0de4095a6436dae99e2be9 Mon Sep 17 00:00:00 2001 From: Jakob Ackermann Date: Wed, 12 Feb 2020 14:39:53 +0100 Subject: [PATCH 319/491] [misc] rename npm-shrinkwrap.json to package-lock.json and run npm i --- services/real-time/{npm-shrinkwrap.json => package-lock.json} | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) rename services/real-time/{npm-shrinkwrap.json => package-lock.json} (99%) diff --git a/services/real-time/npm-shrinkwrap.json b/services/real-time/package-lock.json similarity index 99% rename from services/real-time/npm-shrinkwrap.json rename to services/real-time/package-lock.json index 3b393f9278..fcf5186298 100644 --- a/services/real-time/npm-shrinkwrap.json +++ b/services/real-time/package-lock.json @@ -993,7 +993,7 @@ "finalhandler": { "version": "1.1.1", "resolved": "https://registry.npmjs.org/finalhandler/-/finalhandler-1.1.1.tgz", - "integrity": "sha512-Y1GUDo39ez4aHAw7MysnUD5JzYX+WaIj8I57kO3aEPT1fFRL4sr7mjei97FgnwhAyyzRYmQZaTHb2+9uZ1dPtg==", + "integrity": "sha1-7r9O2EAHnIP0JJA4ydcDAIMBsQU=", "requires": { "debug": "2.6.9", "encodeurl": "~1.0.2", @@ -1007,7 +1007,7 @@ "statuses": { "version": "1.4.0", "resolved": "https://registry.npmjs.org/statuses/-/statuses-1.4.0.tgz", - "integrity": "sha512-zhSCtt8v2NDrRlPQpCNtw/heZLtfUDqxBM1udqikb/Hbk52LK4nQSwr10u77iopCW5LsyHpuXS0GnEc48mLeew==" + "integrity": "sha1-u3PURtonlhBu/MG2AaJT1sRr0Ic=" } } }, From 43013e08204a05b74a3804992500cb044b85bdf3 Mon Sep 17 00:00:00 2001 From: Jakob Ackermann Date: Wed, 12 Feb 2020 14:44:01 +0100 Subject: [PATCH 320/491] [misc] cleanup unused dependency on mongo --- services/real-time/buildscript.txt | 2 +- services/real-time/docker-compose.ci.yml | 4 ---- services/real-time/docker-compose.yml | 5 ----- 3 files changed, 1 insertion(+), 10 deletions(-) diff --git a/services/real-time/buildscript.txt b/services/real-time/buildscript.txt index 8d1c80f036..3e21ebbd0b 100644 --- a/services/real-time/buildscript.txt +++ b/services/real-time/buildscript.txt @@ -4,7 +4,7 @@ real-time --env-add= --node-version=10.19.0 --acceptance-creds=None ---dependencies=mongo,redis +--dependencies=redis --docker-repos=gcr.io/overleaf-ops --env-pass-through= --script-version=1.3.5 diff --git a/services/real-time/docker-compose.ci.yml b/services/real-time/docker-compose.ci.yml index b99da9b18e..292c297cc3 100644 --- a/services/real-time/docker-compose.ci.yml +++ b/services/real-time/docker-compose.ci.yml @@ -25,8 +25,6 @@ services: MOCHA_GREP: ${MOCHA_GREP} NODE_ENV: test depends_on: - mongo: - condition: service_healthy redis: condition: service_healthy user: node @@ -43,5 +41,3 @@ services: redis: image: redis - mongo: - image: mongo:3.6 diff --git a/services/real-time/docker-compose.yml b/services/real-time/docker-compose.yml index 6a1bbb1005..7a856c1ec4 100644 --- a/services/real-time/docker-compose.yml +++ b/services/real-time/docker-compose.yml @@ -32,8 +32,6 @@ services: NODE_ENV: test user: node depends_on: - mongo: - condition: service_healthy redis: condition: service_healthy command: npm run test:acceptance @@ -41,6 +39,3 @@ services: redis: image: redis - mongo: - image: mongo:3.6 - From d320c2d5f368ef3efc5d591434c6c75548e5dccf Mon Sep 17 00:00:00 2001 From: Jakob Ackermann Date: Thu, 20 Feb 2020 21:44:21 +0100 Subject: [PATCH 321/491] [misc] let proxys observe an upcoming shutdown before starting to drain Otherwise clients may be routed to the same pod upon reconnecting. --- services/real-time/app.coffee | 11 ++++++++--- services/real-time/config/settings.defaults.coffee | 2 ++ 2 files changed, 10 insertions(+), 3 deletions(-) diff --git a/services/real-time/app.coffee b/services/real-time/app.coffee index 677737f862..215b3c7cb9 100644 --- a/services/real-time/app.coffee +++ b/services/real-time/app.coffee @@ -125,9 +125,14 @@ drainAndShutdown = (signal) -> return else Settings.shutDownInProgress = true - logger.warn signal: signal, "received interrupt, starting drain over #{shutdownDrainTimeWindow} mins" - DrainManager.startDrainTimeWindow(io, shutdownDrainTimeWindow) - shutdownCleanly(signal) + statusCheckInterval = Settings.statusCheckInterval + if statusCheckInterval + logger.warn signal: signal, "received interrupt, delay drain by #{statusCheckInterval}ms" + setTimeout () -> + logger.warn signal: signal, "received interrupt, starting drain over #{shutdownDrainTimeWindow} mins" + DrainManager.startDrainTimeWindow(io, shutdownDrainTimeWindow) + shutdownCleanly(signal) + , statusCheckInterval Settings.shutDownInProgress = false diff --git a/services/real-time/config/settings.defaults.coffee b/services/real-time/config/settings.defaults.coffee index 75b39ab2ee..5814530552 100644 --- a/services/real-time/config/settings.defaults.coffee +++ b/services/real-time/config/settings.defaults.coffee @@ -60,6 +60,8 @@ settings = publishOnIndividualChannels: process.env['PUBLISH_ON_INDIVIDUAL_CHANNELS'] or false + statusCheckInterval: parseInt(process.env['STATUS_CHECK_INTERVAL'] or '0') + sentry: dsn: process.env.SENTRY_DSN From 15244a54be5a26a6655476703e07759bba1a01ff Mon Sep 17 00:00:00 2001 From: Jakob Ackermann Date: Tue, 24 Mar 2020 09:12:12 +0100 Subject: [PATCH 322/491] [misc] WebsocketController: limit the update size to 7mb bail out early on -- especially do not push the update into redis for doc-updater to discard it. Confirm the update silently, otherwise the frontend will send it again. Broadcast a 'otUpdateError' message and disconnect the client, like doc-updater would do. --- .../app/coffee/WebsocketController.coffee | 18 +++++++ .../real-time/config/settings.defaults.coffee | 5 ++ .../acceptance/coffee/ApplyUpdateTests.coffee | 54 +++++++++++++++++++ .../coffee/WebsocketControllerTests.coffee | 32 +++++++++++ 4 files changed, 109 insertions(+) diff --git a/services/real-time/app/coffee/WebsocketController.coffee b/services/real-time/app/coffee/WebsocketController.coffee index d091c5e4ed..1e4ca5160c 100644 --- a/services/real-time/app/coffee/WebsocketController.coffee +++ b/services/real-time/app/coffee/WebsocketController.coffee @@ -1,5 +1,6 @@ logger = require "logger-sharelatex" metrics = require "metrics-sharelatex" +settings = require "settings-sharelatex" WebApiManager = require "./WebApiManager" AuthorizationManager = require "./AuthorizationManager" DocumentUpdaterManager = require "./DocumentUpdaterManager" @@ -202,6 +203,23 @@ module.exports = WebsocketController = Utils.getClientAttributes client, ["user_id", "project_id"], (error, {user_id, project_id}) -> return callback(error) if error? return callback(new Error("no project_id found on client")) if !project_id? + + updateSize = JSON.stringify(update).length + if updateSize > settings.maxUpdateSize + metrics.inc "update_too_large" + logger.warn({user_id, project_id, doc_id, updateSize}, "update is too large") + + # mark the update as received -- the client should not send it again! + callback() + + # trigger an out-of-sync error + message = {project_id, doc_id, error: "update is too large"} + setTimeout () -> + client.emit "otUpdateError", message.error, message + client.disconnect() + , 100 + return + WebsocketController._assertClientCanApplyUpdate client, doc_id, update, (error) -> if error? logger.warn {err: error, doc_id, client_id: client.id, version: update.v}, "client is not authorized to make update" diff --git a/services/real-time/config/settings.defaults.coffee b/services/real-time/config/settings.defaults.coffee index 5814530552..9ca2348b0b 100644 --- a/services/real-time/config/settings.defaults.coffee +++ b/services/real-time/config/settings.defaults.coffee @@ -52,6 +52,11 @@ settings = max_doc_length: 2 * 1024 * 1024 # 2mb + # combine + # max_doc_length (2mb see above) * 2 (delete + insert) + # max_ranges_size (3mb see MAX_RANGES_SIZE in doc-updater) + maxUpdateSize: parseInt(process.env['MAX_UPDATE_SIZE']) or 7 * 1024 * 1024 + shutdownDrainTimeWindow: process.env['SHUTDOWN_DRAIN_TIME_WINDOW'] or 9 continualPubsubTraffic: process.env['CONTINUAL_PUBSUB_TRAFFIC'] or false diff --git a/services/real-time/test/acceptance/coffee/ApplyUpdateTests.coffee b/services/real-time/test/acceptance/coffee/ApplyUpdateTests.coffee index 842e4fcc4c..2d52975d71 100644 --- a/services/real-time/test/acceptance/coffee/ApplyUpdateTests.coffee +++ b/services/real-time/test/acceptance/coffee/ApplyUpdateTests.coffee @@ -68,6 +68,60 @@ describe "applyOtUpdate", -> (cb) => rclient.del redisSettings.documentupdater.key_schema.pendingUpdates(@doc_id), cb ], done + describe "when authorized with a huge edit update", -> + before (done) -> + @update = { + op: {p: 12, t: "foo"}, + junk: 'this update is too large'.repeat(1024 * 300) # >7MB + } + async.series [ + (cb) => + FixturesManager.setUpProject { + privilegeLevel: "readAndWrite" + }, (e, {@project_id, @user_id}) => + cb(e) + + (cb) => + FixturesManager.setUpDoc @project_id, {@lines, @version, @ops}, (e, {@doc_id}) => + cb(e) + + (cb) => + @client = RealTimeClient.connect() + @client.on "connectionAccepted", cb + @client.on "otUpdateError", (@otUpdateError) => + + (cb) => + @client.emit "joinProject", project_id: @project_id, cb + + (cb) => + @client.emit "joinDoc", @doc_id, cb + + (cb) => + @client.emit "applyOtUpdate", @doc_id, @update, (@error) => + cb() + ], done + + it "should not return an error", -> + expect(@error).to.not.exist + + it "should send an otUpdateError to the client", (done) -> + setTimeout () => + expect(@otUpdateError).to.exist + done() + , 300 + + it "should disconnect the client", (done) -> + setTimeout () => + @client.socket.connected.should.equal false + done() + , 300 + + it "should not put the update in redis", (done) -> + rclient.llen redisSettings.documentupdater.key_schema.pendingUpdates({@doc_id}), (error, len) => + len.should.equal 0 + done() + return null + describe "when authorized to read-only with an edit update", -> before (done) -> async.series [ diff --git a/services/real-time/test/unit/coffee/WebsocketControllerTests.coffee b/services/real-time/test/unit/coffee/WebsocketControllerTests.coffee index f2f1531834..b6b4520ced 100644 --- a/services/real-time/test/unit/coffee/WebsocketControllerTests.coffee +++ b/services/real-time/test/unit/coffee/WebsocketControllerTests.coffee @@ -32,6 +32,7 @@ describe 'WebsocketController', -> "./DocumentUpdaterManager": @DocumentUpdaterManager = {} "./ConnectedUsersManager": @ConnectedUsersManager = {} "./WebsocketLoadBalancer": @WebsocketLoadBalancer = {} + "settings-sharelatex": {maxUpdateSize: 7 * 1024 * 1024} "logger-sharelatex": @logger = { log: sinon.stub(), error: sinon.stub(), warn: sinon.stub() } "metrics-sharelatex": @metrics = inc: sinon.stub() @@ -668,6 +669,37 @@ describe 'WebsocketController', -> it "should call the callback with the error", -> @callback.calledWith(@error).should.equal true + describe "update_too_large", -> + beforeEach (done) -> + @client.disconnect = sinon.stub() + @client.emit = sinon.stub() + @update = { + op: {p: 12, t: "foo"}, + junk: 'this update is too large'.repeat(1024 * 300) # >7MB + } + @client.params.user_id = @user_id + @client.params.project_id = @project_id + @WebsocketController.applyOtUpdate @client, @doc_id, @update, @callback + setTimeout -> + done() + , 201 + + it "should call the callback with no error", -> + @callback.called.should.equal true + @callback.args[0].should.deep.equal [] + + it "should log a warning with the size and context", -> + @logger.warn.called.should.equal true + @logger.warn.args[0].should.deep.equal [{ + @user_id, @project_id, @doc_id, updateSize: 7372835 + }, 'update is too large'] + + it "should send an otUpdateError the client", -> + @client.emit.calledWith('otUpdateError').should.equal true + + it "should disconnect the client", -> + @client.disconnect.called.should.equal true + describe "_assertClientCanApplyUpdate", -> beforeEach -> @edit_update = { op: [{i: "foo", p: 42}, {c: "bar", p: 132}] } # comments may still be in an edit op From cb675d38c2232a23f605683cc32adcbbbfe7bf52 Mon Sep 17 00:00:00 2001 From: Jakob Ackermann Date: Tue, 24 Mar 2020 09:14:15 +0100 Subject: [PATCH 323/491] [misc] SafeJsonParse: align the size limit with the frontend->rt limit frontend -> real-time and doc-updater -> real-time should be in sync. Otherwise we can send a payload to doc-updater, but can not receive the confirmation of it -- and the client will send it again in a loop. Also log the size of the payload. --- services/real-time/app/coffee/SafeJsonParse.coffee | 4 ++-- services/real-time/test/unit/coffee/SafeJsonParseTest.coffee | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/services/real-time/app/coffee/SafeJsonParse.coffee b/services/real-time/app/coffee/SafeJsonParse.coffee index edfa921d74..afeb72f96e 100644 --- a/services/real-time/app/coffee/SafeJsonParse.coffee +++ b/services/real-time/app/coffee/SafeJsonParse.coffee @@ -3,8 +3,8 @@ logger = require "logger-sharelatex" module.exports = parse: (data, callback = (error, parsed) ->) -> - if data.length > (Settings.max_doc_length or 2 * 1024 * 1024) - logger.error {head: data.slice(0,1024)}, "data too large to parse" + if data.length > Settings.maxUpdateSize + logger.error {head: data.slice(0,1024), length: data.length}, "data too large to parse" return callback new Error("data too large to parse") try parsed = JSON.parse(data) diff --git a/services/real-time/test/unit/coffee/SafeJsonParseTest.coffee b/services/real-time/test/unit/coffee/SafeJsonParseTest.coffee index 6a6b5a951c..b652a2faae 100644 --- a/services/real-time/test/unit/coffee/SafeJsonParseTest.coffee +++ b/services/real-time/test/unit/coffee/SafeJsonParseTest.coffee @@ -8,7 +8,7 @@ describe 'SafeJsonParse', -> beforeEach -> @SafeJsonParse = SandboxedModule.require modulePath, requires: "settings-sharelatex": @Settings = { - max_doc_length: 16 * 1024 + maxUpdateSize: 16 * 1024 } "logger-sharelatex": @logger = {error: sinon.stub()} @@ -27,7 +27,7 @@ describe 'SafeJsonParse', -> # we have a 2k overhead on top of max size big_blob = Array(16*1024).join("A") data = "{\"foo\": \"#{big_blob}\"}" - @Settings.max_doc_length = 2 * 1024 + @Settings.maxUpdateSize = 2 * 1024 @SafeJsonParse.parse data, (error, parsed) => @logger.error.called.should.equal true expect(error).to.exist From af53d3b6034bf3dd55a79ecd535fd33e9db93a30 Mon Sep 17 00:00:00 2001 From: Jakob Ackermann Date: Tue, 24 Mar 2020 11:22:28 +0100 Subject: [PATCH 324/491] [misc] skip duplicate JSON serialization for size check --- .../app/coffee/DocumentUpdaterManager.coffee | 8 +++++ .../app/coffee/WebsocketController.coffee | 32 +++++++++---------- .../acceptance/coffee/ApplyUpdateTests.coffee | 6 ++-- .../coffee/DocumentUpdaterManagerTests.coffee | 15 +++++++++ .../coffee/WebsocketControllerTests.coffee | 8 ++--- 5 files changed, 46 insertions(+), 23 deletions(-) diff --git a/services/real-time/app/coffee/DocumentUpdaterManager.coffee b/services/real-time/app/coffee/DocumentUpdaterManager.coffee index 9968b78753..bec80fae34 100644 --- a/services/real-time/app/coffee/DocumentUpdaterManager.coffee +++ b/services/real-time/app/coffee/DocumentUpdaterManager.coffee @@ -61,9 +61,17 @@ module.exports = DocumentUpdaterManager = change = _.pick change, allowedKeys jsonChange = JSON.stringify change if jsonChange.indexOf("\u0000") != -1 + # memory corruption check error = new Error("null bytes found in op") logger.error err: error, project_id: project_id, doc_id: doc_id, jsonChange: jsonChange, error.message return callback(error) + + updateSize = jsonChange.length + if updateSize > settings.maxUpdateSize + error = new Error("update is too large") + error.updateSize = updateSize + return callback(error) + doc_key = "#{project_id}:#{doc_id}" # Push onto pendingUpdates for doc_id first, because once the doc updater # gets an entry on pending-updates-list, it starts processing. diff --git a/services/real-time/app/coffee/WebsocketController.coffee b/services/real-time/app/coffee/WebsocketController.coffee index 1e4ca5160c..786a77d114 100644 --- a/services/real-time/app/coffee/WebsocketController.coffee +++ b/services/real-time/app/coffee/WebsocketController.coffee @@ -204,22 +204,6 @@ module.exports = WebsocketController = return callback(error) if error? return callback(new Error("no project_id found on client")) if !project_id? - updateSize = JSON.stringify(update).length - if updateSize > settings.maxUpdateSize - metrics.inc "update_too_large" - logger.warn({user_id, project_id, doc_id, updateSize}, "update is too large") - - # mark the update as received -- the client should not send it again! - callback() - - # trigger an out-of-sync error - message = {project_id, doc_id, error: "update is too large"} - setTimeout () -> - client.emit "otUpdateError", message.error, message - client.disconnect() - , 100 - return - WebsocketController._assertClientCanApplyUpdate client, doc_id, update, (error) -> if error? logger.warn {err: error, doc_id, client_id: client.id, version: update.v}, "client is not authorized to make update" @@ -236,6 +220,22 @@ module.exports = WebsocketController = logger.log {user_id, doc_id, project_id, client_id: client.id, version: update.v}, "sending update to doc updater" DocumentUpdaterManager.queueChange project_id, doc_id, update, (error) -> + if error?.message == "update is too large" + metrics.inc "update_too_large" + updateSize = error.updateSize + logger.warn({user_id, project_id, doc_id, updateSize}, "update is too large") + + # mark the update as received -- the client should not send it again! + callback() + + # trigger an out-of-sync error + message = {project_id, doc_id, error: "update is too large"} + setTimeout () -> + client.emit "otUpdateError", message.error, message + client.disconnect() + , 100 + return + if error? logger.error {err: error, project_id, doc_id, client_id: client.id, version: update.v}, "document was not available for update" client.disconnect() diff --git a/services/real-time/test/acceptance/coffee/ApplyUpdateTests.coffee b/services/real-time/test/acceptance/coffee/ApplyUpdateTests.coffee index 2d52975d71..7820a090d8 100644 --- a/services/real-time/test/acceptance/coffee/ApplyUpdateTests.coffee +++ b/services/real-time/test/acceptance/coffee/ApplyUpdateTests.coffee @@ -71,8 +71,10 @@ describe "applyOtUpdate", -> describe "when authorized with a huge edit update", -> before (done) -> @update = { - op: {p: 12, t: "foo"}, - junk: 'this update is too large'.repeat(1024 * 300) # >7MB + op: { + p: 12, + t: "update is too large".repeat(1024 * 400) # >7MB + } } async.series [ (cb) => diff --git a/services/real-time/test/unit/coffee/DocumentUpdaterManagerTests.coffee b/services/real-time/test/unit/coffee/DocumentUpdaterManagerTests.coffee index 482d82b0f7..a6bf0a5e47 100644 --- a/services/real-time/test/unit/coffee/DocumentUpdaterManagerTests.coffee +++ b/services/real-time/test/unit/coffee/DocumentUpdaterManagerTests.coffee @@ -15,6 +15,7 @@ describe 'DocumentUpdaterManager', -> redis: documentupdater: key_schema: pendingUpdates: ({doc_id}) -> "PendingUpdates:#{doc_id}" + maxUpdateSize: 7 * 1024 * 1024 @rclient = {auth:->} @DocumentUpdaterManager = SandboxedModule.require modulePath, @@ -163,6 +164,20 @@ describe 'DocumentUpdaterManager', -> it "should not push the change onto the pending-updates-list queue", -> @rclient.rpush.called.should.equal false + describe "when the update is too large", -> + beforeEach -> + @change = {op: {p: 12,t: "update is too large".repeat(1024 * 400)}} + @DocumentUpdaterManager.queueChange(@project_id, @doc_id, @change, @callback) + + it "should return an error", -> + @callback.calledWithExactly(sinon.match(Error)).should.equal true + + it "should add the size to the error", -> + @callback.args[0][0].updateSize.should.equal 7782422 + + it "should not push the change onto the pending-updates-list queue", -> + @rclient.rpush.called.should.equal false + describe "with invalid keys", -> beforeEach -> @change = { diff --git a/services/real-time/test/unit/coffee/WebsocketControllerTests.coffee b/services/real-time/test/unit/coffee/WebsocketControllerTests.coffee index b6b4520ced..62243f797c 100644 --- a/services/real-time/test/unit/coffee/WebsocketControllerTests.coffee +++ b/services/real-time/test/unit/coffee/WebsocketControllerTests.coffee @@ -32,7 +32,6 @@ describe 'WebsocketController', -> "./DocumentUpdaterManager": @DocumentUpdaterManager = {} "./ConnectedUsersManager": @ConnectedUsersManager = {} "./WebsocketLoadBalancer": @WebsocketLoadBalancer = {} - "settings-sharelatex": {maxUpdateSize: 7 * 1024 * 1024} "logger-sharelatex": @logger = { log: sinon.stub(), error: sinon.stub(), warn: sinon.stub() } "metrics-sharelatex": @metrics = inc: sinon.stub() @@ -673,12 +672,11 @@ describe 'WebsocketController', -> beforeEach (done) -> @client.disconnect = sinon.stub() @client.emit = sinon.stub() - @update = { - op: {p: 12, t: "foo"}, - junk: 'this update is too large'.repeat(1024 * 300) # >7MB - } @client.params.user_id = @user_id @client.params.project_id = @project_id + error = new Error("update is too large") + error.updateSize = 7372835 + @DocumentUpdaterManager.queueChange = sinon.stub().callsArgWith(3, error) @WebsocketController.applyOtUpdate @client, @doc_id, @update, @callback setTimeout -> done() From 69569e3571d1e6878959a9afd00de7e8a3cdbb66 Mon Sep 17 00:00:00 2001 From: Jakob Ackermann Date: Tue, 24 Mar 2020 16:21:29 +0100 Subject: [PATCH 325/491] [misc] config: add headroom for JSON serialization in maxUpdateSize --- services/real-time/config/settings.defaults.coffee | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/services/real-time/config/settings.defaults.coffee b/services/real-time/config/settings.defaults.coffee index 9ca2348b0b..ee4bd74316 100644 --- a/services/real-time/config/settings.defaults.coffee +++ b/services/real-time/config/settings.defaults.coffee @@ -54,8 +54,9 @@ settings = # combine # max_doc_length (2mb see above) * 2 (delete + insert) - # max_ranges_size (3mb see MAX_RANGES_SIZE in doc-updater) - maxUpdateSize: parseInt(process.env['MAX_UPDATE_SIZE']) or 7 * 1024 * 1024 + # max_ranges_size (3mb see MAX_RANGES_SIZE in document-updater) + # overhead for JSON serialization + maxUpdateSize: parseInt(process.env['MAX_UPDATE_SIZE']) or 7 * 1024 * 1024 + 64 * 1024 shutdownDrainTimeWindow: process.env['SHUTDOWN_DRAIN_TIME_WINDOW'] or 9 From a9b8e9be3b5478ed58272e56fd54613dc0ef8121 Mon Sep 17 00:00:00 2001 From: Jakob Ackermann Date: Mon, 30 Mar 2020 10:47:01 +0200 Subject: [PATCH 326/491] [misc] upgrade metrics-sharelatex to 2.6.2 --- services/real-time/package-lock.json | 392 ++++++++++++++------------- services/real-time/package.json | 2 +- 2 files changed, 203 insertions(+), 191 deletions(-) diff --git a/services/real-time/package-lock.json b/services/real-time/package-lock.json index fcf5186298..bb15589302 100644 --- a/services/real-time/package-lock.json +++ b/services/real-time/package-lock.json @@ -7,7 +7,7 @@ "@google-cloud/common": { "version": "0.32.1", "resolved": "https://registry.npmjs.org/@google-cloud/common/-/common-0.32.1.tgz", - "integrity": "sha1-ajLDQBcs6j22Z00ODjTnh0CgBz8=", + "integrity": "sha512-bLdPzFvvBMtVkwsoBtygE9oUm3yrNmPa71gvOgucYI/GqvNP2tb6RYsDHPq98kvignhcgHGDI5wyNgxaCo8bKQ==", "requires": { "@google-cloud/projectify": "^0.3.3", "@google-cloud/promisify": "^0.4.0", @@ -25,7 +25,7 @@ "@google-cloud/debug-agent": { "version": "3.2.0", "resolved": "https://registry.npmjs.org/@google-cloud/debug-agent/-/debug-agent-3.2.0.tgz", - "integrity": "sha1-2qdjWhaYpWY31dxXzhED536uKdM=", + "integrity": "sha512-fP87kYbS6aeDna08BivwQ1J260mwJGchRi99XdWCgqbRwuFac8ul0OT5i2wEeDSc5QaDX8ZuWQQ0igZvh1rTyQ==", "requires": { "@google-cloud/common": "^0.32.0", "@sindresorhus/is": "^0.15.0", @@ -44,16 +44,16 @@ }, "dependencies": { "coffeescript": { - "version": "2.4.1", - "resolved": "https://registry.npmjs.org/coffeescript/-/coffeescript-2.4.1.tgz", - "integrity": "sha1-gV/TN98KNNSedKmKbr6pw+eTD3A=" + "version": "2.5.1", + "resolved": "https://registry.npmjs.org/coffeescript/-/coffeescript-2.5.1.tgz", + "integrity": "sha512-J2jRPX0eeFh5VKyVnoLrfVFgLZtnnmp96WQSLAS8OrLm2wtQLcnikYKe1gViJKDH7vucjuhHvBKKBP3rKcD1tQ==" } } }, "@google-cloud/profiler": { "version": "0.2.3", "resolved": "https://registry.npmjs.org/@google-cloud/profiler/-/profiler-0.2.3.tgz", - "integrity": "sha1-Fj3738Mwuug1X+RuHlvgZTV7H1w=", + "integrity": "sha512-rNvtrFtIebIxZEJ/O0t8n7HciZGIXBo8DvHxWqAmsCaeLvkTtsaL6HmPkwxrNQ1IhbYWAxF+E/DwCiHyhKmgTg==", "requires": { "@google-cloud/common": "^0.26.0", "@types/console-log-level": "^1.4.0", @@ -75,7 +75,7 @@ "@google-cloud/common": { "version": "0.26.2", "resolved": "https://registry.npmjs.org/@google-cloud/common/-/common-0.26.2.tgz", - "integrity": "sha1-nFTiRxqEqgMelaJIJJduCA8lVkU=", + "integrity": "sha512-xJ2M/q3MrUbnYZuFlpF01caAlEhAUoRn0NXp93Hn3pkFpfSOG8YfbKbpBAHvcKVbBOAKVIwPsleNtuyuabUwLQ==", "requires": { "@google-cloud/projectify": "^0.3.2", "@google-cloud/promisify": "^0.3.0", @@ -94,7 +94,7 @@ "@google-cloud/promisify": { "version": "0.3.1", "resolved": "https://registry.npmjs.org/@google-cloud/promisify/-/promisify-0.3.1.tgz", - "integrity": "sha1-9kHm2USo4KBe4MsQkd+mAIm+zbo=" + "integrity": "sha512-QzB0/IMvB0eFxFK7Eqh+bfC8NLv3E9ScjWQrPOk6GgfNroxcVITdTlT8NRsRrcp5+QQJVPLkRqKG0PUdaWXmHw==" }, "arrify": { "version": "1.0.1", @@ -104,7 +104,7 @@ "gcp-metadata": { "version": "0.9.3", "resolved": "https://registry.npmjs.org/gcp-metadata/-/gcp-metadata-0.9.3.tgz", - "integrity": "sha1-H510lfdGChRSZIHynhFZbdVj3SY=", + "integrity": "sha512-caV4S84xAjENtpezLCT/GILEAF5h/bC4cNqZFmt/tjTn8t+JBtTkQrgBrJu3857YdsnlM8rxX/PMcKGtE8hUlw==", "requires": { "gaxios": "^1.0.2", "json-bigint": "^0.3.0" @@ -113,7 +113,7 @@ "google-auth-library": { "version": "2.0.2", "resolved": "https://registry.npmjs.org/google-auth-library/-/google-auth-library-2.0.2.tgz", - "integrity": "sha1-ejFdIDZ0Svavyth7IQ7mY4tA9Xs=", + "integrity": "sha512-FURxmo1hBVmcfLauuMRKOPYAPKht3dGuI2wjeJFalDUThO0HoYVjr4yxt5cgYSFm1dgUpmN9G/poa7ceTFAIiA==", "requires": { "axios": "^0.18.0", "gcp-metadata": "^0.7.0", @@ -127,7 +127,7 @@ "gcp-metadata": { "version": "0.7.0", "resolved": "https://registry.npmjs.org/gcp-metadata/-/gcp-metadata-0.7.0.tgz", - "integrity": "sha1-bDXbtSvaMqQnu5yY9UI33dG1QG8=", + "integrity": "sha512-ffjC09amcDWjh3VZdkDngIo7WoluyC5Ag9PAYxZbmQLOLNI8lvPtoKTSCyU54j2gwy5roZh6sSMTfkY2ct7K3g==", "requires": { "axios": "^0.18.0", "extend": "^3.0.1", @@ -137,34 +137,26 @@ } }, "semver": { - "version": "5.7.0", - "resolved": "https://registry.npmjs.org/semver/-/semver-5.7.0.tgz", - "integrity": "sha1-eQp89v6lRZuslhELKbYEEtyP+Ws=" - }, - "through2": { - "version": "3.0.1", - "resolved": "https://registry.npmjs.org/through2/-/through2-3.0.1.tgz", - "integrity": "sha1-OSducTwzAu3544jdnIEt07glvVo=", - "requires": { - "readable-stream": "2 || 3" - } + "version": "5.7.1", + "resolved": "https://registry.npmjs.org/semver/-/semver-5.7.1.tgz", + "integrity": "sha512-sauaDf/PZdVgrLTNYHRtpXa1iRiKcaebiKQ1BJdpQlWH2lCvexQdX55snPFyK7QzpudqbCI0qXFfOasHdyNDGQ==" } } }, "@google-cloud/projectify": { "version": "0.3.3", "resolved": "https://registry.npmjs.org/@google-cloud/projectify/-/projectify-0.3.3.tgz", - "integrity": "sha1-vekQPVCyCj6jM334xng6dm5w1B0=" + "integrity": "sha512-7522YHQ4IhaafgSunsFF15nG0TGVmxgXidy9cITMe+256RgqfcrfWphiMufW+Ou4kqagW/u3yxwbzVEW3dk2Uw==" }, "@google-cloud/promisify": { "version": "0.4.0", "resolved": "https://registry.npmjs.org/@google-cloud/promisify/-/promisify-0.4.0.tgz", - "integrity": "sha1-T7/PTYW7ai5MzwWqY9KxDWyarZs=" + "integrity": "sha512-4yAHDC52TEMCNcMzVC8WlqnKKKq+Ssi2lXoUg9zWWkZ6U6tq9ZBRYLHHCRdfU+EU9YJsVmivwGcKYCjRGjnf4Q==" }, "@google-cloud/trace-agent": { "version": "3.6.1", "resolved": "https://registry.npmjs.org/@google-cloud/trace-agent/-/trace-agent-3.6.1.tgz", - "integrity": "sha1-W+dEE5TQ6ldY8o25IqUAT/PwO+w=", + "integrity": "sha512-KDo85aPN4gSxJ7oEIOlKd7aGENZFXAM1kbIn1Ds+61gh/K1CQWSyepgJo3nUpAwH6D1ezDWV7Iaf8ueoITc8Uw==", "requires": { "@google-cloud/common": "^0.32.1", "builtin-modules": "^3.0.0", @@ -182,9 +174,9 @@ }, "dependencies": { "uuid": { - "version": "3.3.2", - "resolved": "https://registry.npmjs.org/uuid/-/uuid-3.3.2.tgz", - "integrity": "sha1-G0r0lV6zB3xQHCOHL8ZROBFYcTE=" + "version": "3.4.0", + "resolved": "https://registry.npmjs.org/uuid/-/uuid-3.4.0.tgz", + "integrity": "sha512-HjSDRw6gZE5JMggctHBcjVak08+KEVhSIiDzFnT9S9aegmp85S/bReBVTb4QTFaRNptJ9kuYaNhnbNEOkbKb/A==" } } }, @@ -196,12 +188,12 @@ "@protobufjs/base64": { "version": "1.1.2", "resolved": "https://registry.npmjs.org/@protobufjs/base64/-/base64-1.1.2.tgz", - "integrity": "sha1-TIVzDlm5ofHzSQR9vyQpYDS7JzU=" + "integrity": "sha512-AZkcAA5vnN/v4PDqKyMR5lx7hZttPDgClv83E//FMNhR2TMcLUhfRUBHCmSl0oi9zMgDDqRUJkSxO3wm85+XLg==" }, "@protobufjs/codegen": { "version": "2.0.4", "resolved": "https://registry.npmjs.org/@protobufjs/codegen/-/codegen-2.0.4.tgz", - "integrity": "sha1-fvN/DQEPsCitGtWXIuUG2SYoFcs=" + "integrity": "sha512-YyFaikqM5sH0ziFZCN3xDC7zeGaB/d0IUb9CATugHWbd1FRFwWwt4ld4OYMPWu5a3Xe01mGAULCdqhMlPl29Jg==" }, "@protobufjs/eventemitter": { "version": "1.1.0", @@ -245,69 +237,73 @@ "@sindresorhus/is": { "version": "0.15.0", "resolved": "https://registry.npmjs.org/@sindresorhus/is/-/is-0.15.0.tgz", - "integrity": "sha1-lpFbqgXmpqHRN7rfSYTT/AWCC7Y=" + "integrity": "sha512-lu8BpxjAtRCAo5ifytTpCPCj99LF7o/2Myn+NXyNCBqvPYn7Pjd76AMmUB5l7XF1U6t0hcWrlEM5ESufW7wAeA==" }, "@types/caseless": { "version": "0.12.2", "resolved": "https://registry.npmjs.org/@types/caseless/-/caseless-0.12.2.tgz", - "integrity": "sha1-9l09Y4ngHutFi9VNyPUrlalGO8g=" + "integrity": "sha512-6ckxMjBBD8URvjB6J3NcnuAn5Pkl7t3TizAg+xdlzzQGSPSmBcXf8KoIH0ua/i+tio+ZRUHEXp0HEmvaR4kt0w==" }, "@types/console-log-level": { "version": "1.4.0", "resolved": "https://registry.npmjs.org/@types/console-log-level/-/console-log-level-1.4.0.tgz", - "integrity": "sha1-7/ccQa689RyLpa2LBdfVQkviuPM=" + "integrity": "sha512-x+OscEQwcx5Biair4enH7ov9W+clcqUWaZRaxn5IkT4yNWWjRr2oiYDkY/x1uXSTVZOQ2xlbFQySaQGB+VdXGQ==" }, "@types/duplexify": { "version": "3.6.0", "resolved": "https://registry.npmjs.org/@types/duplexify/-/duplexify-3.6.0.tgz", - "integrity": "sha1-38grZL06IWj1vSZESvFlvwI33Ng=", - "requires": { - "@types/node": "*" - } - }, - "@types/form-data": { - "version": "2.2.1", - "resolved": "https://registry.npmjs.org/@types/form-data/-/form-data-2.2.1.tgz", - "integrity": "sha1-7is7jqoRwJOCiZU2BrdFtzjFSx4=", + "integrity": "sha512-5zOA53RUlzN74bvrSGwjudssD9F3a797sDZQkiYpUOxW+WHaXTCPz4/d5Dgi6FKnOqZ2CpaTo0DhgIfsXAOE/A==", "requires": { "@types/node": "*" } }, "@types/long": { - "version": "4.0.0", - "resolved": "https://registry.npmjs.org/@types/long/-/long-4.0.0.tgz", - "integrity": "sha1-cZVR0jUtMBrIuB23Mqy2vcKNve8=" + "version": "4.0.1", + "resolved": "https://registry.npmjs.org/@types/long/-/long-4.0.1.tgz", + "integrity": "sha512-5tXH6Bx/kNGd3MgffdmP4dy2Z+G4eaXw0SE81Tq3BNadtnMR5/ySMzX4SLEzHJzSmPNn4HIdpQsBvXMUykr58w==" }, "@types/node": { - "version": "12.0.8", - "resolved": "https://registry.npmjs.org/@types/node/-/node-12.0.8.tgz", - "integrity": "sha1-VRRmvhGyrcPz1HFWdY9hC9n2sdg=" + "version": "13.9.5", + "resolved": "https://registry.npmjs.org/@types/node/-/node-13.9.5.tgz", + "integrity": "sha512-hkzMMD3xu6BrJpGVLeQ3htQQNAcOrJjX7WFmtK8zWQpz2UJf13LCFF2ALA7c9OVdvc2vQJeDdjfR35M0sBCxvw==" }, "@types/request": { - "version": "2.48.1", - "resolved": "https://registry.npmjs.org/@types/request/-/request-2.48.1.tgz", - "integrity": "sha1-5ALWkapmcPu/8ZV7FfEnAjCrQvo=", + "version": "2.48.4", + "resolved": "https://registry.npmjs.org/@types/request/-/request-2.48.4.tgz", + "integrity": "sha512-W1t1MTKYR8PxICH+A4HgEIPuAC3sbljoEVfyZbeFJJDbr30guDspJri2XOaM2E+Un7ZjrihaDi7cf6fPa2tbgw==", "requires": { "@types/caseless": "*", - "@types/form-data": "*", "@types/node": "*", - "@types/tough-cookie": "*" + "@types/tough-cookie": "*", + "form-data": "^2.5.0" + }, + "dependencies": { + "form-data": { + "version": "2.5.1", + "resolved": "https://registry.npmjs.org/form-data/-/form-data-2.5.1.tgz", + "integrity": "sha512-m21N3WOmEEURgk6B9GLOE4RuWOFf28Lhh9qGYeNlGq4VDXUlJy2th2slBNU8Gp8EzloYZOibZJ7t5ecIrFSjVA==", + "requires": { + "asynckit": "^0.4.0", + "combined-stream": "^1.0.6", + "mime-types": "^2.1.12" + } + } } }, "@types/semver": { "version": "5.5.0", "resolved": "https://registry.npmjs.org/@types/semver/-/semver-5.5.0.tgz", - "integrity": "sha1-FGwqKe59O65L8vyydGNuJkyBPEU=" + "integrity": "sha512-41qEJgBH/TWgo5NFSvBCJ1qkoi3Q6ONSF2avrHq1LVEZfYpdHmj0y9SuTK+u9ZhG1sYQKBL1AWXKyLWP4RaUoQ==" }, "@types/tough-cookie": { - "version": "2.3.5", - "resolved": "https://registry.npmjs.org/@types/tough-cookie/-/tough-cookie-2.3.5.tgz", - "integrity": "sha1-naRO11VxmZtlw3tgybK4jbVMWF0=" + "version": "2.3.6", + "resolved": "https://registry.npmjs.org/@types/tough-cookie/-/tough-cookie-2.3.6.tgz", + "integrity": "sha512-wHNBMnkoEBiRAd3s8KTKwIuO9biFtTf0LehITzBhSco+HQI0xkXZbLOD55SW3Aqw3oUkHstkm5SPv58yaAdFPQ==" }, "abort-controller": { "version": "3.0.0", "resolved": "https://registry.npmjs.org/abort-controller/-/abort-controller-3.0.0.tgz", - "integrity": "sha1-6vVNU7YrrkE46AnKIlyEOabvs5I=", + "integrity": "sha512-h8lQ8tacZYnR3vNQTgibj+tODHI5/+l06Au2Pcriv/Gmet0eaj4TwWH41sO9wnHDiQsEj19q0drzdWdeAHtweg==", "requires": { "event-target-shim": "^5.0.0" } @@ -322,9 +318,9 @@ } }, "acorn": { - "version": "6.1.1", - "resolved": "https://registry.npmjs.org/acorn/-/acorn-6.1.1.tgz", - "integrity": "sha1-fSWuBbuK0fm2mRCOEJTs14hK3B8=" + "version": "6.4.1", + "resolved": "https://registry.npmjs.org/acorn/-/acorn-6.4.1.tgz", + "integrity": "sha512-ZVA9k326Nwrj3Cj9jlh3wGFutC2ZornPNARZwsNYqQYgN0EsV2d53w5RN/co65Ohn4sUAUtb1rSUAOD6XN9idA==" }, "active-x-obfuscator": { "version": "0.0.1", @@ -337,7 +333,7 @@ "agent-base": { "version": "4.3.0", "resolved": "https://registry.npmjs.org/agent-base/-/agent-base-4.3.0.tgz", - "integrity": "sha1-gWXwHENgCbzK0LHRIvBe13Dvxu4=", + "integrity": "sha512-salcGninV0nPrwpGNn4VTXBb1SOuXQBiqbrNXoeizJsHrsL6ERFM2Ne3JUSBWRE6aeNJI2ROP/WEEIDUiDe3cg==", "requires": { "es6-promisify": "^5.0.0" } @@ -361,7 +357,7 @@ "arrify": { "version": "2.0.1", "resolved": "https://registry.npmjs.org/arrify/-/arrify-2.0.1.tgz", - "integrity": "sha1-yWVekzHgq81YjSp8rX6ZVvZnAfo=" + "integrity": "sha512-3duEwti880xqi4eAMN8AyR4a0ByT90zoYdLlevfrvU43vb0YZwZVfxOgxWrLXXXpyugL0hNZc9G6BiB5B3nUug==" }, "assert-plus": { "version": "1.0.0", @@ -382,16 +378,16 @@ "async-listener": { "version": "0.6.10", "resolved": "https://registry.npmjs.org/async-listener/-/async-listener-0.6.10.tgz", - "integrity": "sha1-p8l6vlcLpgLXgic8DeYKUePhfLw=", + "integrity": "sha512-gpuo6xOyF4D5DE5WvyqZdPA3NGhiT6Qf07l7DCB0wwDEsLvDIbCr6j9S5aj5Ch96dLace5tXVzWBZkxU/c5ohw==", "requires": { "semver": "^5.3.0", "shimmer": "^1.1.0" }, "dependencies": { "semver": { - "version": "5.7.0", - "resolved": "https://registry.npmjs.org/semver/-/semver-5.7.0.tgz", - "integrity": "sha1-eQp89v6lRZuslhELKbYEEtyP+Ws=" + "version": "5.7.1", + "resolved": "https://registry.npmjs.org/semver/-/semver-5.7.1.tgz", + "integrity": "sha512-sauaDf/PZdVgrLTNYHRtpXa1iRiKcaebiKQ1BJdpQlWH2lCvexQdX55snPFyK7QzpudqbCI0qXFfOasHdyNDGQ==" } } }, @@ -413,7 +409,7 @@ "axios": { "version": "0.18.1", "resolved": "https://registry.npmjs.org/axios/-/axios-0.18.1.tgz", - "integrity": "sha1-/z8N4ue10YDnV62YAA8Qgbh7zqM=", + "integrity": "sha512-0BfJq4NSfQXd+SkFdrvFbG7addhYSBA2mQwISr46pD6E5iqkWg02RAs8vyTT/j0RTnoYmeXauBuSv1qKwR179g==", "requires": { "follow-redirects": "1.5.10", "is-buffer": "^2.0.2" @@ -425,9 +421,9 @@ "integrity": "sha1-ibTRmasr7kneFk6gK4nORi1xt2c=" }, "base64-js": { - "version": "1.3.0", - "resolved": "https://registry.npmjs.org/base64-js/-/base64-js-1.3.0.tgz", - "integrity": "sha1-yrHmEY8FEJXli1KBrqjBzSK/wOM=" + "version": "1.3.1", + "resolved": "https://registry.npmjs.org/base64-js/-/base64-js-1.3.1.tgz", + "integrity": "sha512-mLQ4i2QO1ytvGWFWmcngKO//JXAQueZvwEKtjgQFM4jIK0kU+ytMfplL8j+n5mspOfjHwoAg+9yhb7BwAHm36g==" }, "base64-url": { "version": "1.2.1", @@ -456,12 +452,12 @@ "bignumber.js": { "version": "7.2.1", "resolved": "https://registry.npmjs.org/bignumber.js/-/bignumber.js-7.2.1.tgz", - "integrity": "sha1-gMBIdZ2CaACAfEv9Uh5Q7bulel8=" + "integrity": "sha512-S4XzBk5sMB+Rcb/LNcpzXr57VRTxgAvaAEDAl1AwRx27j00hT84O6OkteE7u8UB3NuaaygCRrEpqox4uDOrbdQ==" }, "bindings": { "version": "1.5.0", "resolved": "https://registry.npmjs.org/bindings/-/bindings-1.5.0.tgz", - "integrity": "sha1-EDU8npRTNLwFEabZCzj7x8nFBN8=", + "integrity": "sha512-p2q/t/mhvuOj/UeLlV6566GD/guowlr0hHxClI0W9m7MWYkL1F0hLo+0Aexs9HSPCtR1SXQ0TD3MMKrXZajbiQ==", "requires": { "file-uri-to-path": "1.0.0" } @@ -511,7 +507,7 @@ "builtin-modules": { "version": "3.1.0", "resolved": "https://registry.npmjs.org/builtin-modules/-/builtin-modules-3.1.0.tgz", - "integrity": "sha1-qtl8FRMet2tltQ7yCOdYTNdqdIQ=" + "integrity": "sha512-k0KL0aWZuBt2lrxrcASWDfwOLMnodeQjodT/1SxEQAXsHANgo6ZC/VEaSEHCXt7aSTZ4/4H5LKa+tBXmW7Vtvw==" }, "bunyan": { "version": "0.22.3", @@ -588,7 +584,7 @@ "console-log-level": { "version": "1.4.1", "resolved": "https://registry.npmjs.org/console-log-level/-/console-log-level-1.4.1.tgz", - "integrity": "sha1-nFprue8e9lsFq6gwKLD/iUzfYwo=" + "integrity": "sha512-VZzbIORbP+PPcN/gg3DXClTLPLg5Slwd5fL2MIc+o1qZ4BXBvWyc6QxPk6T/Mkr6IVjRpoAGf32XxP3ZWMVRcQ==" }, "content-disposition": { "version": "0.5.2", @@ -603,7 +599,7 @@ "continuation-local-storage": { "version": "3.2.1", "resolved": "https://registry.npmjs.org/continuation-local-storage/-/continuation-local-storage-3.2.1.tgz", - "integrity": "sha1-EfYT906RT+mzTJKtLSj+auHbf/s=", + "integrity": "sha512-jx44cconVqkCEEyLSKWwkvUXwO561jXMa3LPjTPsm5QR22PA0/mhe33FT4Xb5y74JDvt/Cq+5lm8S8rskLv9ZA==", "requires": { "async-listener": "^0.6.0", "emitter-listener": "^1.1.1" @@ -682,7 +678,7 @@ "delay": { "version": "4.3.0", "resolved": "https://registry.npmjs.org/delay/-/delay-4.3.0.tgz", - "integrity": "sha1-7+6/uPVFV5yzlrOnIkQ+yW0UxQ4=" + "integrity": "sha512-Lwaf3zVFDMBop1yDuFZ19F9WyGcZcGacsbdlZtWjQmM50tOcMntm1njF/Nb/Vjij3KaSvCF+sEYGKrrjObu2NA==" }, "delayed-stream": { "version": "1.0.0", @@ -714,7 +710,7 @@ "duplexify": { "version": "3.7.1", "resolved": "https://registry.npmjs.org/duplexify/-/duplexify-3.7.1.tgz", - "integrity": "sha1-Kk31MX9sz9kfhtb9JdjYoQO4gwk=", + "integrity": "sha512-07z8uv2wMyS51kKhD1KsdXJg5WQ6t93RneqRxUHnskXVtlYYkLqM0gqStQZ3pj073g687jPCHrqNfCzawLYh5g==", "requires": { "end-of-stream": "^1.0.0", "inherits": "^2.0.1", @@ -734,7 +730,7 @@ "ecdsa-sig-formatter": { "version": "1.0.11", "resolved": "https://registry.npmjs.org/ecdsa-sig-formatter/-/ecdsa-sig-formatter-1.0.11.tgz", - "integrity": "sha1-rg8PothQRe8UqBfao86azQSJ5b8=", + "integrity": "sha512-nagl3RYrbNv6kQkeJIpt6NJZy8twLB/2vtz6yN9Z4vRKHN4/QZJIEbqohALSgwKdnksuY3k5Addp5lg8sVoVcQ==", "requires": { "safe-buffer": "^5.0.1" } @@ -747,7 +743,7 @@ "emitter-listener": { "version": "1.1.2", "resolved": "https://registry.npmjs.org/emitter-listener/-/emitter-listener-1.1.2.tgz", - "integrity": "sha1-VrFA6PaZI3Wz18ssqxzHQy2WMug=", + "integrity": "sha512-Bt1sBAGFHY9DKY+4/2cV6izcKJUf5T7/gkdmkxzX/qv9CcGH8xSwVRW5mtX03SWJtRTWSOpzCuWN9rBFYZepZQ==", "requires": { "shimmer": "^1.2.0" } @@ -758,9 +754,9 @@ "integrity": "sha1-rT/0yG7C0CkyL1oCw6mmBslbP1k=" }, "end-of-stream": { - "version": "1.4.1", - "resolved": "https://registry.npmjs.org/end-of-stream/-/end-of-stream-1.4.1.tgz", - "integrity": "sha1-7SljTRm6ukY7bOa4CjchPqtx7EM=", + "version": "1.4.4", + "resolved": "https://registry.npmjs.org/end-of-stream/-/end-of-stream-1.4.4.tgz", + "integrity": "sha512-+uw1inIHVPQoaVuHzRyXd21icM+cnt4CzD5rW+NC1wjOUSTOs+Te7FOv7AhN7vS9x/oIyhLP5PR1H+phQAHu5Q==", "requires": { "once": "^1.4.0" } @@ -798,7 +794,7 @@ "es6-promise": { "version": "4.2.8", "resolved": "https://registry.npmjs.org/es6-promise/-/es6-promise-4.2.8.tgz", - "integrity": "sha1-TrIVlMlyvEBVPSduUQU5FD21Pgo=" + "integrity": "sha512-HJDGx5daxeIvxdBxvG2cb9g4tEvwIk3i8+nhX0yGrYmZUzbkdg8QbDevheDB8gd0//uPj4c1EQua8Q+MViT0/w==" }, "es6-promisify": { "version": "5.0.0", @@ -827,7 +823,7 @@ "event-target-shim": { "version": "5.0.1", "resolved": "https://registry.npmjs.org/event-target-shim/-/event-target-shim-5.0.1.tgz", - "integrity": "sha1-XU0+vflYPWOlMzzi3rdICrKwV4k=" + "integrity": "sha512-i/2XbnSz/uxRCU6+NdVJgKWDTM427+MqYbkQzD321DuCQJUqOuJKIA0IM2+W2xtYHdKOmZ4dR6fExsd4SXL+WQ==" }, "express": { "version": "4.16.3", @@ -981,14 +977,14 @@ "integrity": "sha1-1RQsDK7msRifh9OnYREGT4bIu/I=" }, "fast-text-encoding": { - "version": "1.0.0", - "resolved": "https://registry.npmjs.org/fast-text-encoding/-/fast-text-encoding-1.0.0.tgz", - "integrity": "sha1-PlzoKTQJz6pxd6cbnKhOGx5vJe8=" + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/fast-text-encoding/-/fast-text-encoding-1.0.1.tgz", + "integrity": "sha512-x4FEgaz3zNRtJfLFqJmHWxkMDDvXVtaznj2V9jiP8ACUJrUgist4bP9FmDL2Vew2Y9mEQI/tG4GqabaitYp9CQ==" }, "file-uri-to-path": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/file-uri-to-path/-/file-uri-to-path-1.0.0.tgz", - "integrity": "sha1-VTp7hEb/b2hDWcRF8eN6BdrMM90=" + "integrity": "sha512-0Zt+s3L7Vf1biwWZ29aARiVYLx7iMGnEUl9x33fbB/j3jR81u/O2LbqK+Bm1CDSNDKVtJ/YjwY7TUd5SkeLQLw==" }, "finalhandler": { "version": "1.1.1", @@ -1019,7 +1015,7 @@ "follow-redirects": { "version": "1.5.10", "resolved": "https://registry.npmjs.org/follow-redirects/-/follow-redirects-1.5.10.tgz", - "integrity": "sha1-e3qfmuov3/NnhqlP9kPtB/T/Xio=", + "integrity": "sha512-0V5l4Cizzvqt5D44aTXbFZz+FtyXV1vrDN6qrelxtfYQKW0KO0W2T/hkE8xvGa/540LkZlkaUjO4ailYTFtHVQ==", "requires": { "debug": "=3.1.0" }, @@ -1027,7 +1023,7 @@ "debug": { "version": "3.1.0", "resolved": "https://registry.npmjs.org/debug/-/debug-3.1.0.tgz", - "integrity": "sha1-W7WgZyYotkFJVmuhaBnmFRjGcmE=", + "integrity": "sha512-OX8XqP7/1a9cqkxYw2yXss15f26NKWBpDXQd0/uK/KPqdQhxbPa994hnzjcE2VqQpDslf55723cKPUOGSmMY3g==", "requires": { "ms": "2.0.0" } @@ -1083,7 +1079,7 @@ "gaxios": { "version": "1.8.4", "resolved": "https://registry.npmjs.org/gaxios/-/gaxios-1.8.4.tgz", - "integrity": "sha1-4Iw0/pPAqbZ6Ure556ZOZDX5ozk=", + "integrity": "sha512-BoENMnu1Gav18HcpV9IleMPZ9exM+AvUjrAOV4Mzs/vfz2Lu/ABv451iEXByKiMPn2M140uul1txXCg83sAENw==", "requires": { "abort-controller": "^3.0.0", "extend": "^3.0.2", @@ -1094,7 +1090,7 @@ "gcp-metadata": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/gcp-metadata/-/gcp-metadata-1.0.0.tgz", - "integrity": "sha1-UhJEAin6CZ/C98KlzcuVV16bLKY=", + "integrity": "sha512-Q6HrgfrCQeEircnNP3rCcEgiDv7eF9+1B+1MMgpE190+/+0mjQR8PxeOaRgxZWmdDAF9EIryHB9g1moPiw1SbQ==", "requires": { "gaxios": "^1.0.2", "json-bigint": "^0.3.0" @@ -1131,7 +1127,7 @@ "google-auth-library": { "version": "3.1.2", "resolved": "https://registry.npmjs.org/google-auth-library/-/google-auth-library-3.1.2.tgz", - "integrity": "sha1-/y+IzVzSEYpXvT1a08CTyIN/w1A=", + "integrity": "sha512-cDQMzTotwyWMrg5jRO7q0A4TL/3GWBgO7I7q5xGKNiiFf9SmGY/OJ1YsLMgI2MVHHsEGyrqYnbnmV1AE+Z6DnQ==", "requires": { "base64-js": "^1.3.0", "fast-text-encoding": "^1.0.0", @@ -1145,16 +1141,16 @@ }, "dependencies": { "semver": { - "version": "5.7.0", - "resolved": "https://registry.npmjs.org/semver/-/semver-5.7.0.tgz", - "integrity": "sha1-eQp89v6lRZuslhELKbYEEtyP+Ws=" + "version": "5.7.1", + "resolved": "https://registry.npmjs.org/semver/-/semver-5.7.1.tgz", + "integrity": "sha512-sauaDf/PZdVgrLTNYHRtpXa1iRiKcaebiKQ1BJdpQlWH2lCvexQdX55snPFyK7QzpudqbCI0qXFfOasHdyNDGQ==" } } }, "google-p12-pem": { "version": "1.0.4", "resolved": "https://registry.npmjs.org/google-p12-pem/-/google-p12-pem-1.0.4.tgz", - "integrity": "sha1-t3+4M6Lrn388aJ4uVPCVJ293dgU=", + "integrity": "sha512-SwLAUJqUfTB2iS+wFfSS/G9p7bt4eWcc2LyfvmUXe7cWp6p3mpxDo6LLI29MXdU6wvPcQ/up298X7GMC5ylAlA==", "requires": { "node-forge": "^0.8.0", "pify": "^4.0.0" @@ -1163,7 +1159,7 @@ "gtoken": { "version": "2.3.3", "resolved": "https://registry.npmjs.org/gtoken/-/gtoken-2.3.3.tgz", - "integrity": "sha1-in/hVcXODEtxyIbPsoKpBg2UpkE=", + "integrity": "sha512-EaB49bu/TCoNeQjhCYKI/CurooBKkGxIqFHsWABW0b25fobBYVTMe84A8EBVVZhl8emiUdNypil9huMOTmyAnw==", "requires": { "gaxios": "^1.0.4", "google-p12-pem": "^1.0.0", @@ -1175,7 +1171,7 @@ "mime": { "version": "2.4.4", "resolved": "https://registry.npmjs.org/mime/-/mime-2.4.4.tgz", - "integrity": "sha1-vXuRE1/GsBzePpuuM9ZZtj2IV+U=" + "integrity": "sha512-LRxmNwziLPT828z+4YkNzloCFC2YM4wrB99k+AV5ZbEyfGNWfG8SO1FUXLmLDBSo89NrJZ4DIWeLjy1CHGhMGA==" } } }, @@ -1217,7 +1213,7 @@ "hex2dec": { "version": "1.1.2", "resolved": "https://registry.npmjs.org/hex2dec/-/hex2dec-1.1.2.tgz", - "integrity": "sha1-jhzkvvNqdPfVcjw/swkMKGAHczg=" + "integrity": "sha512-Yu+q/XWr2fFQ11tHxPq4p4EiNkb2y+lAacJNhAdRXVfRIcDH6gi7htWFnnlIzvqHMHoWeIsfXlNAjZInpAOJDA==" }, "http-errors": { "version": "1.6.3", @@ -1241,18 +1237,18 @@ } }, "https-proxy-agent": { - "version": "2.2.1", - "resolved": "https://registry.npmjs.org/https-proxy-agent/-/https-proxy-agent-2.2.1.tgz", - "integrity": "sha1-UVUpcPoE1yPgTFbQQXjD+SWSu8A=", + "version": "2.2.4", + "resolved": "https://registry.npmjs.org/https-proxy-agent/-/https-proxy-agent-2.2.4.tgz", + "integrity": "sha512-OmvfoQ53WLjtA9HeYP9RNrWMJzzAz1JGaSFr1nijg0PVR1JaD/xbJq1mdEIIlxGpXp9eSe/O2LgU9DJmTPd0Eg==", "requires": { - "agent-base": "^4.1.0", + "agent-base": "^4.3.0", "debug": "^3.1.0" }, "dependencies": { "debug": { "version": "3.2.6", "resolved": "https://registry.npmjs.org/debug/-/debug-3.2.6.tgz", - "integrity": "sha1-6D0X3hbYp++3cX7b5fsQE17uYps=", + "integrity": "sha512-mel+jf7nrtEl5Pn1Qx46zARXKDpBbvzezse7p7LqINmdoIk8PYP5SySaxEmYv6TZ0JyEKA1hsCId6DIhgITtWQ==", "requires": { "ms": "^2.1.1" } @@ -1260,7 +1256,7 @@ "ms": { "version": "2.1.2", "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.2.tgz", - "integrity": "sha1-0J0fNXtEP0kzgqjrPM0YOHKuYAk=" + "integrity": "sha512-sGkPx+VjMtmA6MX27oA4FBFELFCZZ4S4XqeGOXCv68tT+jb3vk/RyaKWP0PTKyWtmLSM0b+adUTEvbs1PEaH2w==" } } }, @@ -1325,7 +1321,7 @@ "is": { "version": "3.3.0", "resolved": "https://registry.npmjs.org/is/-/is-3.3.0.tgz", - "integrity": "sha1-Yc/23TxBk9uUo9YlggcrROVkXXk=" + "integrity": "sha512-nW24QBoPcFGGHJGUwnfpI7Yc5CdqWNdsyHQszVE/z2pKHXzh7FZ5GWhJqSyaQ9wMkQnsTx+kAI8bHlCX4tKdbg==" }, "is-arguments": { "version": "1.0.4", @@ -1334,9 +1330,9 @@ "dev": true }, "is-buffer": { - "version": "2.0.3", - "resolved": "https://registry.npmjs.org/is-buffer/-/is-buffer-2.0.3.tgz", - "integrity": "sha1-Ts8/z3ScvR5HJonhCaxmJhol5yU=" + "version": "2.0.4", + "resolved": "https://registry.npmjs.org/is-buffer/-/is-buffer-2.0.4.tgz", + "integrity": "sha512-Kq1rokWXOPXWuaMAqZiJW4XxsmD9zGx9q4aePabbn3qCRGedtH7Cm+zV8WETitMfu1wdh+Rvd6w5egwSngUX2A==" }, "is-callable": { "version": "1.1.4", @@ -1438,7 +1434,7 @@ "jwa": { "version": "1.4.1", "resolved": "https://registry.npmjs.org/jwa/-/jwa-1.4.1.tgz", - "integrity": "sha1-dDwymFy56YZVUw1TZBtmyGRbA5o=", + "integrity": "sha512-qiLX/xhEEFKUAJ6FiBMbes3w9ATzyk5W7Hvzpa/SLYdxNtng+gcurvrI7TbACjIXlsJyr05/S1oUhZrc63evQA==", "requires": { "buffer-equal-constant-time": "1.0.1", "ecdsa-sig-formatter": "1.0.11", @@ -1448,7 +1444,7 @@ "jws": { "version": "3.2.2", "resolved": "https://registry.npmjs.org/jws/-/jws-3.2.2.tgz", - "integrity": "sha1-ABCZ82OUaMlBQADpmZX6UvtHgwQ=", + "integrity": "sha512-YHlZCB6lMTllWDtSPHz/ZXTsi8S00usEV6v1tjq8tOUZzw7DpSDWVXjXDre6ed1w/pd495ODpHZYSdkRTsa0HA==", "requires": { "jwa": "^1.4.1", "safe-buffer": "^5.0.1" @@ -1515,12 +1511,12 @@ "long": { "version": "4.0.0", "resolved": "https://registry.npmjs.org/long/-/long-4.0.0.tgz", - "integrity": "sha1-mntxz7fTYaGU6lVSQckvdGjVvyg=" + "integrity": "sha512-XsP+KhQif4bjX1kbuSiySJFNAehNxgLb6hPRGJ9QsUr8ajHkuXGdrHmFUTUUXhDwVX2R5bY4JNZEwbUiMhV+MA==" }, "lru-cache": { "version": "5.1.1", "resolved": "https://registry.npmjs.org/lru-cache/-/lru-cache-5.1.1.tgz", - "integrity": "sha1-HaJ+ZxAnGUdpXa9oSOhH8B2EuSA=", + "integrity": "sha512-KpNARQA3Iwv+jTA0utUVVbrh+Jlrr1Fv0e56GGzAFOXN7dk/FviaDW8LHmK52DlcH4WP2n6gI8vN1aesBFgo9w==", "requires": { "yallist": "^3.0.2" } @@ -1560,9 +1556,9 @@ "integrity": "sha1-VSmk1nZUE07cxSZmVoNbD4Ua/O4=" }, "metrics-sharelatex": { - "version": "2.2.0", - "resolved": "https://registry.npmjs.org/metrics-sharelatex/-/metrics-sharelatex-2.2.0.tgz", - "integrity": "sha1-RM9oy9FuUQYgfrZ+PvkAhaQWwqk=", + "version": "2.6.2", + "resolved": "https://registry.npmjs.org/metrics-sharelatex/-/metrics-sharelatex-2.6.2.tgz", + "integrity": "sha512-bOLfkSCexiPgB96hdXhoOWyvvrwscgjeZPEqdcJ7BTGxY59anzvymNf5hTGJ1RtS4sblDKxITw3L5a+gYKhRYQ==", "requires": { "@google-cloud/debug-agent": "^3.0.0", "@google-cloud/profiler": "^0.2.3", @@ -1570,7 +1566,8 @@ "coffee-script": "1.6.0", "lynx": "~0.1.1", "prom-client": "^11.1.3", - "underscore": "~1.6.0" + "underscore": "~1.6.0", + "yn": "^3.1.1" }, "dependencies": { "underscore": { @@ -1747,12 +1744,12 @@ "node-fetch": { "version": "2.6.0", "resolved": "https://registry.npmjs.org/node-fetch/-/node-fetch-2.6.0.tgz", - "integrity": "sha1-5jNFY4bUqlWGP2dqerDaqP3ssP0=" + "integrity": "sha512-8dG4H5ujfvFiqDmVu9fQ5bOHUC15JMjMY/Zumv26oOvvVJjM67KF8koCWIabKQ1GJIa9r2mMZscBq/TbdOcmNA==" }, "node-forge": { - "version": "0.8.4", - "resolved": "https://registry.npmjs.org/node-forge/-/node-forge-0.8.4.tgz", - "integrity": "sha1-1nOGYrZhvhnicR7wGqOxghLxMDA=" + "version": "0.8.5", + "resolved": "https://registry.npmjs.org/node-forge/-/node-forge-0.8.5.tgz", + "integrity": "sha512-vFMQIWt+J/7FLNyKouZ9TazT74PRV3wgv9UT4cRjC8BffxFbKXkgIWR42URCPSnHm/QDz6BOlb2Q0U4+VQT67Q==" }, "oauth-sign": { "version": "0.9.0", @@ -1804,9 +1801,9 @@ "integrity": "sha1-7CLTEoBrtT5zF3Pnza788cZDEo8=" }, "p-limit": { - "version": "2.2.0", - "resolved": "https://registry.npmjs.org/p-limit/-/p-limit-2.2.0.tgz", - "integrity": "sha1-QXyZQeYCepq8ulCS3SkE4lW1+8I=", + "version": "2.2.2", + "resolved": "https://registry.npmjs.org/p-limit/-/p-limit-2.2.2.tgz", + "integrity": "sha512-WGR+xHecKTr7EbUEhyLSh5Dube9JtdiG78ufaeLxTgpudf/20KqyMioIUZJAezlTIi6evxuoUs9YXc11cU+yzQ==", "requires": { "p-try": "^2.0.0" } @@ -1814,17 +1811,17 @@ "p-try": { "version": "2.2.0", "resolved": "https://registry.npmjs.org/p-try/-/p-try-2.2.0.tgz", - "integrity": "sha1-yyhoVA4xPWHeWPr741zpAE1VQOY=" + "integrity": "sha512-R4nPAVTAU0B9D35/Gk3uJf/7XYbQcyohSKdvAxIRSNghFl4e71hVoGnBNQz9cWaXxO2I10KTC+3jMdvvoKw6dQ==" }, "parse-duration": { - "version": "0.1.1", - "resolved": "https://registry.npmjs.org/parse-duration/-/parse-duration-0.1.1.tgz", - "integrity": "sha1-ExFN3JiRwezSgANiRFVN5DZHoiY=" + "version": "0.1.2", + "resolved": "https://registry.npmjs.org/parse-duration/-/parse-duration-0.1.2.tgz", + "integrity": "sha512-0qfMZyjOUFBeEIvJ5EayfXJqaEXxQ+Oj2b7tWJM3hvEXvXsYCk05EDVI23oYnEw2NaFYUWdABEVPBvBMh8L/pA==" }, "parse-ms": { "version": "2.1.0", "resolved": "https://registry.npmjs.org/parse-ms/-/parse-ms-2.1.0.tgz", - "integrity": "sha1-NIVlp1PUOR+lJAKZVrFyy3dTCX0=" + "integrity": "sha512-kHt7kzLoS9VBZfUsiKjv43mr91ea+U05EyKkEtqp7vNbHxmaVuEqN7XxeEVnGrMtYOAxGrDElSi96K7EgO1zCA==" }, "parseurl": { "version": "1.3.2", @@ -1839,7 +1836,7 @@ "path-parse": { "version": "1.0.6", "resolved": "https://registry.npmjs.org/path-parse/-/path-parse-1.0.6.tgz", - "integrity": "sha1-1i27VnlAXXLEc37FhgDp3c8G0kw=" + "integrity": "sha512-GSmOT2EbHrINBf9SR7CDELwlJ8AENk3Qn7OikK4nFYAu3Ote2+JYNVvkpAEQm3/TLNEJFD/xZJjzyxg3KBWOzw==" }, "path-to-regexp": { "version": "0.1.7", @@ -1854,7 +1851,7 @@ "pify": { "version": "4.0.1", "resolved": "https://registry.npmjs.org/pify/-/pify-4.0.1.tgz", - "integrity": "sha1-SyzSXFDVmHNcUCkiJP2MbfQeMjE=" + "integrity": "sha512-uB80kBFb/tfd68bVleG9T5GGsGPjJrLAUpR5PZIrhBnIaRTQRjqdJSsIKkOP6OAIFbj7GOrcudc5pNjZ+geV2g==" }, "policyfile": { "version": "0.0.4", @@ -1864,28 +1861,28 @@ "pretty-ms": { "version": "4.0.0", "resolved": "https://registry.npmjs.org/pretty-ms/-/pretty-ms-4.0.0.tgz", - "integrity": "sha1-Mbr0G5T9AiJwmKqgO9YmCOsNbpI=", + "integrity": "sha512-qG66ahoLCwpLXD09ZPHSCbUWYTqdosB7SMP4OffgTgL2PBKXMuUsrk5Bwg8q4qPkjTXsKBMr+YK3Ltd/6F9s/Q==", "requires": { "parse-ms": "^2.0.0" } }, "process-nextick-args": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/process-nextick-args/-/process-nextick-args-2.0.0.tgz", - "integrity": "sha1-o31zL0JxtKsa0HDTVQjoKQeI/6o=" + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/process-nextick-args/-/process-nextick-args-2.0.1.tgz", + "integrity": "sha512-3ouUOpQhtgrbOa17J7+uxOTpITYWaGP7/AhoR3+A+/1e9skrzelGi/dXzEYyvbxubEF6Wn2ypscTKiKJFFn1ag==" }, "prom-client": { - "version": "11.5.1", - "resolved": "https://registry.npmjs.org/prom-client/-/prom-client-11.5.1.tgz", - "integrity": "sha1-FcZsrN7EUwELz68EEJvMNOa92pw=", + "version": "11.5.3", + "resolved": "https://registry.npmjs.org/prom-client/-/prom-client-11.5.3.tgz", + "integrity": "sha512-iz22FmTbtkyL2vt0MdDFY+kWof+S9UB/NACxSn2aJcewtw+EERsen0urSkZ2WrHseNdydsvcxCTAnPcSMZZv4Q==", "requires": { "tdigest": "^0.1.1" } }, "protobufjs": { - "version": "6.8.8", - "resolved": "https://registry.npmjs.org/protobufjs/-/protobufjs-6.8.8.tgz", - "integrity": "sha1-yLTxKC/XqQ5vWxCe0RyEr4KQjnw=", + "version": "6.8.9", + "resolved": "https://registry.npmjs.org/protobufjs/-/protobufjs-6.8.9.tgz", + "integrity": "sha512-j2JlRdUeL/f4Z6x4aU4gj9I2LECglC+5qR2TrWb193Tla1qfdaNQTZ8I27Pt7K0Ajmvjjpft7O3KWTGciz4gpw==", "requires": { "@protobufjs/aspromise": "^1.1.2", "@protobufjs/base64": "^1.1.2", @@ -1903,9 +1900,9 @@ }, "dependencies": { "@types/node": { - "version": "10.14.9", - "resolved": "https://registry.npmjs.org/@types/node/-/node-10.14.9.tgz", - "integrity": "sha1-Lo1ngDnSeUPOU6GRM4YTMif9kGY=" + "version": "10.17.17", + "resolved": "https://registry.npmjs.org/@types/node/-/node-10.17.17.tgz", + "integrity": "sha512-gpNnRnZP3VWzzj5k3qrpRC6Rk3H/uclhAVo1aIvwzK5p5cOrs9yEyQ8H/HBsBY0u5rrWxXEiVPQ0dEB6pkjE8Q==" } } }, @@ -1972,9 +1969,9 @@ } }, "readable-stream": { - "version": "2.3.6", - "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-2.3.6.tgz", - "integrity": "sha1-sRwn2IuP8fvgcGQ8+UsMea4bCq8=", + "version": "2.3.7", + "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-2.3.7.tgz", + "integrity": "sha512-Ebho8K4jIbHAxnuxi7o42OrZgF/ZTNcsZj6nRKyUmkhLFq8CHItp/fy6hQZuZmP/n3yZ9VBUbp4zz/mX8hmYPw==", "requires": { "core-util-is": "~1.0.0", "inherits": "~2.0.3", @@ -2112,19 +2109,19 @@ } }, "require-in-the-middle": { - "version": "4.0.0", - "resolved": "https://registry.npmjs.org/require-in-the-middle/-/require-in-the-middle-4.0.0.tgz", - "integrity": "sha1-PHUoik7EgM30S8d950T4q+WFQFs=", + "version": "4.0.1", + "resolved": "https://registry.npmjs.org/require-in-the-middle/-/require-in-the-middle-4.0.1.tgz", + "integrity": "sha512-EfkM2zANyGkrfIExsECMeNn/uzjvHrE9h36yLXSavmrDiH4tgDNvltAmEKnt4PNLbqKPHZz+uszW2wTKrLUX0w==", "requires": { "debug": "^4.1.1", "module-details-from-path": "^1.0.3", - "resolve": "^1.10.0" + "resolve": "^1.12.0" }, "dependencies": { "debug": { "version": "4.1.1", "resolved": "https://registry.npmjs.org/debug/-/debug-4.1.1.tgz", - "integrity": "sha1-O3ImAlUQnGtYnO4FDx1RYTlmR5E=", + "integrity": "sha512-pYAIzeRo8J6KPEaJ0VWOh5Pzkbw/RetuzehGM7QRRX5he4fPHx2rdKMB256ehJCkX+XRQm16eZLqLNS8RSZXZw==", "requires": { "ms": "^2.1.1" } @@ -2132,7 +2129,7 @@ "ms": { "version": "2.1.2", "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.2.tgz", - "integrity": "sha1-0J0fNXtEP0kzgqjrPM0YOHKuYAk=" + "integrity": "sha512-sGkPx+VjMtmA6MX27oA4FBFELFCZZ4S4XqeGOXCv68tT+jb3vk/RyaKWP0PTKyWtmLSM0b+adUTEvbs1PEaH2w==" } } }, @@ -2143,9 +2140,9 @@ "dev": true }, "resolve": { - "version": "1.11.0", - "resolved": "https://registry.npmjs.org/resolve/-/resolve-1.11.0.tgz", - "integrity": "sha1-QBSHC6KWF2uGND1Qtg87UGCc4jI=", + "version": "1.15.1", + "resolved": "https://registry.npmjs.org/resolve/-/resolve-1.15.1.tgz", + "integrity": "sha512-84oo6ZTtoTUpjgNEr5SJyzQhzL72gaRodsSfyxC/AXRvwu0Yse9H8eF9IpGo7b8YetZhlI6v7ZQ6bKBFV/6S7w==", "requires": { "path-parse": "^1.0.6" } @@ -2153,14 +2150,30 @@ "retry-axios": { "version": "0.3.2", "resolved": "https://registry.npmjs.org/retry-axios/-/retry-axios-0.3.2.tgz", - "integrity": "sha1-V1fID1hbTMTEmGqi/9R6YMbTXhM=" + "integrity": "sha512-jp4YlI0qyDFfXiXGhkCOliBN1G7fRH03Nqy8YdShzGqbY5/9S2x/IR6C88ls2DFkbWuL3ASkP7QD3pVrNpPgwQ==" }, "retry-request": { - "version": "4.0.0", - "resolved": "https://registry.npmjs.org/retry-request/-/retry-request-4.0.0.tgz", - "integrity": "sha1-XDZhZiebPhDp16oTJ0RnoFy2kpA=", + "version": "4.1.1", + "resolved": "https://registry.npmjs.org/retry-request/-/retry-request-4.1.1.tgz", + "integrity": "sha512-BINDzVtLI2BDukjWmjAIRZ0oglnCAkpP2vQjM3jdLhmT62h0xnQgciPwBRDAvHqpkPT2Wo1XuUyLyn6nbGrZQQ==", "requires": { - "through2": "^2.0.0" + "debug": "^4.1.1", + "through2": "^3.0.1" + }, + "dependencies": { + "debug": { + "version": "4.1.1", + "resolved": "https://registry.npmjs.org/debug/-/debug-4.1.1.tgz", + "integrity": "sha512-pYAIzeRo8J6KPEaJ0VWOh5Pzkbw/RetuzehGM7QRRX5he4fPHx2rdKMB256ehJCkX+XRQm16eZLqLNS8RSZXZw==", + "requires": { + "ms": "^2.1.1" + } + }, + "ms": { + "version": "2.1.2", + "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.2.tgz", + "integrity": "sha512-sGkPx+VjMtmA6MX27oA4FBFELFCZZ4S4XqeGOXCv68tT+jb3vk/RyaKWP0PTKyWtmLSM0b+adUTEvbs1PEaH2w==" + } } }, "rimraf": { @@ -2213,9 +2226,9 @@ } }, "semver": { - "version": "6.1.1", - "resolved": "https://registry.npmjs.org/semver/-/semver-6.1.1.tgz", - "integrity": "sha1-U/U9qbMLIQPNTxXqs6GOy8shDJs=" + "version": "6.3.0", + "resolved": "https://registry.npmjs.org/semver/-/semver-6.3.0.tgz", + "integrity": "sha512-b39TBaTSfV6yBrapU89p5fKekE2m/NwnDocOVruQFS1/veMgdzuPcnOM34M6CwxW8jH/lxEa5rBoDeUwu5HHTw==" }, "send": { "version": "0.16.2", @@ -2283,7 +2296,7 @@ "shimmer": { "version": "1.2.1", "resolved": "https://registry.npmjs.org/shimmer/-/shimmer-1.2.1.tgz", - "integrity": "sha1-YQhZ994ye1h+/r9QH7QxF/mv8zc=" + "integrity": "sha512-sQTKC1Re/rM6XyFM6fIAGHRPVGvyXfgzIDvzoq608vM+jeyVD0Tu1E6Np0Kc2zAIFWIj963V2800iF/9LPieQw==" }, "sinon": { "version": "1.17.7", @@ -2341,12 +2354,12 @@ "source-map": { "version": "0.6.1", "resolved": "https://registry.npmjs.org/source-map/-/source-map-0.6.1.tgz", - "integrity": "sha1-dHIq8y6WFOnCh6jQu95IteLxomM=" + "integrity": "sha512-UjgapumWlbMhkBgzT7Ykc5YXUT46F0iKu8SGXq0bcwP5dz/h0Plj6enJqjz1Zbq2l5WaqYnrVbwWOWMyF3F47g==" }, "split": { "version": "1.0.1", "resolved": "https://registry.npmjs.org/split/-/split-1.0.1.tgz", - "integrity": "sha1-YFvZvjA6pZ+zX5Ip++oN3snqB9k=", + "integrity": "sha512-mTyOoPbrivtXnwnIxZRFYRrPNtEFKlpB2fvjSnCQUiAA6qAZzqwna5envK4uk6OIeP17CsdF3rSBGYVBsU0Tkg==", "requires": { "through": "2" } @@ -2403,14 +2416,14 @@ "integrity": "sha1-Fhx9rBd2Wf2YEfQ3cfqZOBR4Yow=" }, "stream-shift": { - "version": "1.0.0", - "resolved": "https://registry.npmjs.org/stream-shift/-/stream-shift-1.0.0.tgz", - "integrity": "sha1-1cdSgl5TZ+eG944Y5EXqIjoVWVI=" + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/stream-shift/-/stream-shift-1.0.1.tgz", + "integrity": "sha512-AiisoFqQ0vbGcZgQPY1cdP2I76glaVA/RauYR4G4thNFgkTqr90yXTo4LYX60Jl+sIlPNHHdGSwo01AvbKUSVQ==" }, "string_decoder": { "version": "1.1.1", "resolved": "https://registry.npmjs.org/string_decoder/-/string_decoder-1.1.1.tgz", - "integrity": "sha1-nPFhG6YmhdcDCunkujQUnDrwP8g=", + "integrity": "sha512-n/ShnvDi6FHbbVfviro+WojiFzv+s8MPMHBczVePfUpDJLwoLT0ht1l4YwBCbi8pJAveEEdnkHyPyTP/mzRfwg==", "requires": { "safe-buffer": "~5.1.0" } @@ -2426,7 +2439,7 @@ "teeny-request": { "version": "3.11.3", "resolved": "https://registry.npmjs.org/teeny-request/-/teeny-request-3.11.3.tgz", - "integrity": "sha1-M1xin3ZF5dZZk2LfLzIwxMvCOlU=", + "integrity": "sha512-CKncqSF7sH6p4rzCgkb/z/Pcos5efl0DmolzvlqRQUNcpRIruOhY9+T1FsIlyEbfWd7MsFpodROOwHYh2BaXzw==", "requires": { "https-proxy-agent": "^2.2.1", "node-fetch": "^2.2.0", @@ -2434,9 +2447,9 @@ }, "dependencies": { "uuid": { - "version": "3.3.2", - "resolved": "https://registry.npmjs.org/uuid/-/uuid-3.3.2.tgz", - "integrity": "sha1-G0r0lV6zB3xQHCOHL8ZROBFYcTE=" + "version": "3.4.0", + "resolved": "https://registry.npmjs.org/uuid/-/uuid-3.4.0.tgz", + "integrity": "sha512-HjSDRw6gZE5JMggctHBcjVak08+KEVhSIiDzFnT9S9aegmp85S/bReBVTb4QTFaRNptJ9kuYaNhnbNEOkbKb/A==" } } }, @@ -2446,12 +2459,11 @@ "integrity": "sha1-DdTJ/6q8NXlgsbckEV1+Doai4fU=" }, "through2": { - "version": "2.0.5", - "resolved": "https://registry.npmjs.org/through2/-/through2-2.0.5.tgz", - "integrity": "sha1-AcHjnrMdB8t9A6lqcIIyYLIxMs0=", + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/through2/-/through2-3.0.1.tgz", + "integrity": "sha512-M96dvTalPT3YbYLaKaCuwu+j06D/8Jfib0o/PxbVt6Amhv3dUAtW6rTV1jPgJSBG83I/e04Y6xkVdVhSRhi0ww==", "requires": { - "readable-stream": "~2.3.6", - "xtend": "~4.0.1" + "readable-stream": "2 || 3" } }, "timekeeper": { @@ -2633,15 +2645,15 @@ "resolved": "https://registry.npmjs.org/xmlhttprequest/-/xmlhttprequest-1.4.2.tgz", "integrity": "sha1-AUU6HZvtHo8XL2SVu/TIxCYyFQA=" }, - "xtend": { - "version": "4.0.1", - "resolved": "https://registry.npmjs.org/xtend/-/xtend-4.0.1.tgz", - "integrity": "sha1-pcbVMr5lbiPbgg77lDofBJmNY68=" - }, "yallist": { - "version": "3.0.3", - "resolved": "https://registry.npmjs.org/yallist/-/yallist-3.0.3.tgz", - "integrity": "sha1-tLBJ4xS+VF486AIjbWzSLNkcPek=" + "version": "3.1.1", + "resolved": "https://registry.npmjs.org/yallist/-/yallist-3.1.1.tgz", + "integrity": "sha512-a4UGQaWPH59mOXUYnAG2ewncQS4i4F43Tv3JoAM+s2VDAmS9NsK8GpDMLrCHPksFT7h3K6TOoUNn2pb7RoXx4g==" + }, + "yn": { + "version": "3.1.1", + "resolved": "https://registry.npmjs.org/yn/-/yn-3.1.1.tgz", + "integrity": "sha512-Ux4ygGWsu2c7isFWe8Yu1YluJmqVhxqK2cLXNQA5AcC3QfbGNpM7fu0Y8b/z16pXLnFxZYvWhd3fhBY9DLmC6Q==" }, "zeparser": { "version": "0.0.5", diff --git a/services/real-time/package.json b/services/real-time/package.json index e6c630b662..21ae57c1d6 100644 --- a/services/real-time/package.json +++ b/services/real-time/package.json @@ -29,7 +29,7 @@ "express": "^4.10.1", "express-session": "^1.9.1", "logger-sharelatex": "^1.7.0", - "metrics-sharelatex": "^2.2.0", + "metrics-sharelatex": "^2.6.2", "redis-sharelatex": "^1.0.11", "request": "^2.88.0", "session.socket.io": "^0.1.6", From 56628a16c632507bfd5775cfa02f369876ce4658 Mon Sep 17 00:00:00 2001 From: Jakob Ackermann Date: Mon, 30 Mar 2020 11:31:44 +0200 Subject: [PATCH 327/491] [misc] track redis pub/sub payload sizes on publish --- services/real-time/app.coffee | 5 ++++- services/real-time/app/coffee/ChannelManager.coffee | 1 + .../test/unit/coffee/ChannelManagerTests.coffee | 12 +++++++++++- 3 files changed, 16 insertions(+), 2 deletions(-) diff --git a/services/real-time/app.coffee b/services/real-time/app.coffee index 215b3c7cb9..06fd3555f6 100644 --- a/services/real-time/app.coffee +++ b/services/real-time/app.coffee @@ -163,10 +163,13 @@ if Settings.continualPubsubTraffic checker = new HealthCheckManager(channel) logger.debug {channel:channel}, "sending pub to keep connection alive" json = JSON.stringify({health_check:true, key: checker.id, date: new Date().toString()}) + Metrics.summary "redis.publish.#{channel}", json.length pubsubClient.publish channel, json, (err)-> if err? logger.err {err, channel}, "error publishing pubsub traffic to redis" - clusterClient.publish "cluster-continual-traffic", {keep: "alive"}, callback + blob = JSON.stringify({keep: "alive"}) + Metrics.summary "redis.publish.cluster-continual-traffic", blob.length + clusterClient.publish "cluster-continual-traffic", blob, callback runPubSubTraffic = -> diff --git a/services/real-time/app/coffee/ChannelManager.coffee b/services/real-time/app/coffee/ChannelManager.coffee index 3ea5c2e71e..367d2059a2 100644 --- a/services/real-time/app/coffee/ChannelManager.coffee +++ b/services/real-time/app/coffee/ChannelManager.coffee @@ -48,6 +48,7 @@ module.exports = ChannelManager = metrics.inc "unsubscribe.#{baseChannel}" publish: (rclient, baseChannel, id, data) -> + metrics.summary "redis.publish.#{baseChannel}", data.length if id is 'all' or !settings.publishOnIndividualChannels channel = baseChannel else diff --git a/services/real-time/test/unit/coffee/ChannelManagerTests.coffee b/services/real-time/test/unit/coffee/ChannelManagerTests.coffee index e550e963d4..edde3e8170 100644 --- a/services/real-time/test/unit/coffee/ChannelManagerTests.coffee +++ b/services/real-time/test/unit/coffee/ChannelManagerTests.coffee @@ -10,7 +10,7 @@ describe 'ChannelManager', -> @other_rclient = {} @ChannelManager = SandboxedModule.require modulePath, requires: "settings-sharelatex": @settings = {} - "metrics-sharelatex": @metrics = {inc: sinon.stub()} + "metrics-sharelatex": @metrics = {inc: sinon.stub(), summary: sinon.stub()} "logger-sharelatex": @logger = { log: sinon.stub(), warn: sinon.stub(), error: sinon.stub() } describe "subscribe", -> @@ -106,3 +106,13 @@ describe 'ChannelManager', -> @rclient.publish.calledWithExactly("applied-ops:1234567890abcdef", "random-message").should.equal true @rclient.publish.calledOnce.should.equal true + describe "metrics", -> + beforeEach -> + @rclient.publish = sinon.stub() + @ChannelManager.publish @rclient, "applied-ops", "all", "random-message" + + it "should track the payload size", -> + @metrics.summary.calledWithExactly( + "redis.publish.applied-ops", + "random-message".length + ).should.equal true From 8711abdb6609b00e62e56c8d4ba5847e45d0b461 Mon Sep 17 00:00:00 2001 From: Henry Oswald Date: Tue, 31 Mar 2020 14:04:33 +0100 Subject: [PATCH 328/491] bump redis to 1.0.12 --- services/real-time/package-lock.json | 30 ++++++++++++++-------------- services/real-time/package.json | 2 +- 2 files changed, 16 insertions(+), 16 deletions(-) diff --git a/services/real-time/package-lock.json b/services/real-time/package-lock.json index fcf5186298..1d2c2c15e7 100644 --- a/services/real-time/package-lock.json +++ b/services/real-time/package-lock.json @@ -546,7 +546,7 @@ "cluster-key-slot": { "version": "1.1.0", "resolved": "https://registry.npmjs.org/cluster-key-slot/-/cluster-key-slot-1.1.0.tgz", - "integrity": "sha1-MEdLKpgfsSFyaVgzBSvA0BM20Q0=" + "integrity": "sha512-2Nii8p3RwAPiFwsnZvukotvow2rIHM+yQ6ZcBXGHdniadkYGZYiGmkHJIbZPIV9nfv7m/U1IPMVVcAhoWFeklw==" }, "coffee-script": { "version": "1.6.0", @@ -692,7 +692,7 @@ "denque": { "version": "1.4.1", "resolved": "https://registry.npmjs.org/denque/-/denque-1.4.1.tgz", - "integrity": "sha1-Z0T/dkHBSMP4ppwwflEjXB9KN88=" + "integrity": "sha512-OfzPuSZKGcgr96rf1oODnfjqBFmr1DVoc/TrItj3Ohe0Ah1C5WX5Baquw/9U9KovnQ88EqmJbD66rKYUQYN1tQ==" }, "depd": { "version": "1.1.2", @@ -1287,9 +1287,9 @@ "integrity": "sha1-Yzwsg+PaQqUC9SRmAiSA9CCCYd4=" }, "ioredis": { - "version": "4.14.1", - "resolved": "https://registry.npmjs.org/ioredis/-/ioredis-4.14.1.tgz", - "integrity": "sha1-tz3tlfzyIPEG0zElqS72ITqjExg=", + "version": "4.16.1", + "resolved": "https://registry.npmjs.org/ioredis/-/ioredis-4.16.1.tgz", + "integrity": "sha512-g76Mm9dE7BLuewncu1MimGZw5gDDjDwjoRony/VoSxSJEKAhuYncDEwYKYjtHi2NWsTNIB6XXRjE64uVa/wpKQ==", "requires": { "cluster-key-slot": "^1.1.0", "debug": "^4.1.1", @@ -1305,7 +1305,7 @@ "debug": { "version": "4.1.1", "resolved": "https://registry.npmjs.org/debug/-/debug-4.1.1.tgz", - "integrity": "sha1-O3ImAlUQnGtYnO4FDx1RYTlmR5E=", + "integrity": "sha512-pYAIzeRo8J6KPEaJ0VWOh5Pzkbw/RetuzehGM7QRRX5he4fPHx2rdKMB256ehJCkX+XRQm16eZLqLNS8RSZXZw==", "requires": { "ms": "^2.1.1" } @@ -1313,7 +1313,7 @@ "ms": { "version": "2.1.2", "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.2.tgz", - "integrity": "sha1-0J0fNXtEP0kzgqjrPM0YOHKuYAk=" + "integrity": "sha512-sGkPx+VjMtmA6MX27oA4FBFELFCZZ4S4XqeGOXCv68tT+jb3vk/RyaKWP0PTKyWtmLSM0b+adUTEvbs1PEaH2w==" } } }, @@ -1457,7 +1457,7 @@ "lodash": { "version": "4.17.15", "resolved": "https://registry.npmjs.org/lodash/-/lodash-4.17.15.tgz", - "integrity": "sha1-tEf2ZwoEVbv+7dETku/zMOoJdUg=" + "integrity": "sha512-8xOcRHvCjnocdS5cpwXQXVzmmh5e5+saE2QGoeQmbKmRS6J3VQppPOIt0MnmE+4xlZoumy0GPG0D0MVIQbNA1A==" }, "lodash.defaults": { "version": "4.2.0", @@ -1993,7 +1993,7 @@ "redis-commands": { "version": "1.5.0", "resolved": "https://registry.npmjs.org/redis-commands/-/redis-commands-1.5.0.tgz", - "integrity": "sha1-gNLiBpj+aI8icSf/nlFkp90X54U=" + "integrity": "sha512-6KxamqpZ468MeQC3bkWmCB1fp56XL64D4Kf0zJSwDZbVLLm7KFkoIcHrgRvQ+sk8dnhySs7+yBg94yIkAK7aJg==" }, "redis-errors": { "version": "1.2.0", @@ -2025,13 +2025,13 @@ } }, "redis-sharelatex": { - "version": "1.0.11", - "resolved": "https://registry.npmjs.org/redis-sharelatex/-/redis-sharelatex-1.0.11.tgz", - "integrity": "sha1-fcTlS2/FHrsh3WjJcOgdIMPEaMM=", + "version": "1.0.12", + "resolved": "https://registry.npmjs.org/redis-sharelatex/-/redis-sharelatex-1.0.12.tgz", + "integrity": "sha512-Z+LDGaRNgZ+NiDaCC/R0N3Uy6SCtbKXqiXlvCwAbIQRSZUc69OVx/cQ3i5qDF7zeERhh+pnTd+zGs8nVfa5p+Q==", "requires": { "async": "^2.5.0", "coffee-script": "1.8.0", - "ioredis": "~4.14.1", + "ioredis": "~4.16.1", "redis-sentinel": "0.1.1", "underscore": "1.7.0" }, @@ -2039,7 +2039,7 @@ "async": { "version": "2.6.3", "resolved": "https://registry.npmjs.org/async/-/async-2.6.3.tgz", - "integrity": "sha1-1yYl4jRKNlbjo61Pp0n6gymdgv8=", + "integrity": "sha512-zflvls11DCy+dQWzTW2dzuilv8Z5X/pjfmZOWba6TNIVDm+2UDaJmXSOXlasHKfNBs8oo3M0aT50fDEWfKZjXg==", "requires": { "lodash": "^4.17.14" } @@ -2390,7 +2390,7 @@ "standard-as-callback": { "version": "2.0.1", "resolved": "https://registry.npmjs.org/standard-as-callback/-/standard-as-callback-2.0.1.tgz", - "integrity": "sha1-7YuyVkjhWDF1m2Ajvbh+a2CzgSY=" + "integrity": "sha512-NQOxSeB8gOI5WjSaxjBgog2QFw55FV8TkS6Y07BiB3VJ8xNTvUYm0wl0s8ObgQ5NhdpnNfigMIKjgPESzgr4tg==" }, "statsd-parser": { "version": "0.0.4", diff --git a/services/real-time/package.json b/services/real-time/package.json index e6c630b662..fb0c1cb853 100644 --- a/services/real-time/package.json +++ b/services/real-time/package.json @@ -30,7 +30,7 @@ "express-session": "^1.9.1", "logger-sharelatex": "^1.7.0", "metrics-sharelatex": "^2.2.0", - "redis-sharelatex": "^1.0.11", + "redis-sharelatex": "^1.0.12", "request": "^2.88.0", "session.socket.io": "^0.1.6", "settings-sharelatex": "^1.1.0", From 5c28da10310a2fdebbb65209651105635561f5b4 Mon Sep 17 00:00:00 2001 From: Brian Gough Date: Mon, 6 Apr 2020 16:24:10 +0100 Subject: [PATCH 329/491] add metric for pendingUpdates queue --- services/real-time/app/coffee/DocumentUpdaterManager.coffee | 3 +++ .../test/unit/coffee/DocumentUpdaterManagerTests.coffee | 1 + 2 files changed, 4 insertions(+) diff --git a/services/real-time/app/coffee/DocumentUpdaterManager.coffee b/services/real-time/app/coffee/DocumentUpdaterManager.coffee index bec80fae34..c5c5a67cb7 100644 --- a/services/real-time/app/coffee/DocumentUpdaterManager.coffee +++ b/services/real-time/app/coffee/DocumentUpdaterManager.coffee @@ -72,6 +72,9 @@ module.exports = DocumentUpdaterManager = error.updateSize = updateSize return callback(error) + # record metric for each update added to queue + metrics.summary 'redis.pendingUpdates', updateSize, {status: 'push'} + doc_key = "#{project_id}:#{doc_id}" # Push onto pendingUpdates for doc_id first, because once the doc updater # gets an entry on pending-updates-list, it starts processing. diff --git a/services/real-time/test/unit/coffee/DocumentUpdaterManagerTests.coffee b/services/real-time/test/unit/coffee/DocumentUpdaterManagerTests.coffee index a6bf0a5e47..aa4600d757 100644 --- a/services/real-time/test/unit/coffee/DocumentUpdaterManagerTests.coffee +++ b/services/real-time/test/unit/coffee/DocumentUpdaterManagerTests.coffee @@ -25,6 +25,7 @@ describe 'DocumentUpdaterManager', -> 'request': @request = {} 'redis-sharelatex' : createClient: () => @rclient 'metrics-sharelatex': @Metrics = + summary: sinon.stub() Timer: class Timer done: () -> globals: From 684cb3903c45127f8ca687b687b46b0aded03357 Mon Sep 17 00:00:00 2001 From: Jakob Ackermann Date: Tue, 12 May 2020 12:45:46 +0200 Subject: [PATCH 330/491] [WebsocketController] handle redis subscribe error on joinProject joinProject should not complete when the redis pub/sub subscribe request failed. --- .../app/coffee/WebsocketController.coffee | 1 + .../coffee/WebsocketControllerTests.coffee | 22 +++++++++++++++++++ 2 files changed, 23 insertions(+) diff --git a/services/real-time/app/coffee/WebsocketController.coffee b/services/real-time/app/coffee/WebsocketController.coffee index 786a77d114..e9017b05b9 100644 --- a/services/real-time/app/coffee/WebsocketController.coffee +++ b/services/real-time/app/coffee/WebsocketController.coffee @@ -40,6 +40,7 @@ module.exports = WebsocketController = client.set("is_restricted_user", !!(isRestrictedUser)) RoomManager.joinProject client, project_id, (err) -> + return callback(err) if err logger.log {user_id, project_id, client_id: client.id}, "user joined project" callback null, project, privilegeLevel, WebsocketController.PROTOCOL_VERSION diff --git a/services/real-time/test/unit/coffee/WebsocketControllerTests.coffee b/services/real-time/test/unit/coffee/WebsocketControllerTests.coffee index 62243f797c..c6d29d5715 100644 --- a/services/real-time/test/unit/coffee/WebsocketControllerTests.coffee +++ b/services/real-time/test/unit/coffee/WebsocketControllerTests.coffee @@ -125,6 +125,28 @@ describe 'WebsocketController', -> it "should not log an error", -> @logger.error.called.should.equal false + describe "when the subscribe failed", -> + beforeEach -> + @client.id = "mock-client-id" + @project = { + name: "Test Project" + owner: { + _id: @owner_id = "mock-owner-id-123" + } + } + @privilegeLevel = "owner" + @ConnectedUsersManager.updateUserPosition = sinon.stub().callsArg(4) + @isRestrictedUser = true + @WebApiManager.joinProject = sinon.stub().callsArgWith(2, null, @project, @privilegeLevel, @isRestrictedUser) + @RoomManager.joinProject = sinon.stub().callsArgWith(2, new Error("subscribe failed")) + @WebsocketController.joinProject @client, @user, @project_id, @callback + + it "should return an error", -> + @callback + .calledWith(new Error("subscribe failed")) + .should.equal true + @callback.args[0][0].message.should.equal "subscribe failed" + describe "leaveProject", -> beforeEach -> @DocumentUpdaterManager.flushProjectToMongoAndDelete = sinon.stub().callsArg(1) From 55af5e502ffed9ba2d41ef515a840aa28cf19af8 Mon Sep 17 00:00:00 2001 From: Jakob Ackermann Date: Tue, 12 May 2020 12:55:04 +0200 Subject: [PATCH 331/491] [WebsocketController] skip leaveProject when joinProject didn't complete Also drop dead code: - user_id bailout There is a check on a completed joinProject call now. It will always set a user_id, see Router.coffee which has a fallback `{_id:"..."}`. - late project_id bailout WebsocketLoadBalancer.emitToRoom will not work without a project_id. We have to bail out before the call. --- .../app/coffee/WebsocketController.coffee | 13 ++---------- .../coffee/WebsocketControllerTests.coffee | 21 +++++++++++++++---- 2 files changed, 19 insertions(+), 15 deletions(-) diff --git a/services/real-time/app/coffee/WebsocketController.coffee b/services/real-time/app/coffee/WebsocketController.coffee index 786a77d114..35f572c856 100644 --- a/services/real-time/app/coffee/WebsocketController.coffee +++ b/services/real-time/app/coffee/WebsocketController.coffee @@ -51,23 +51,14 @@ module.exports = WebsocketController = # is determined by FLUSH_IF_EMPTY_DELAY. FLUSH_IF_EMPTY_DELAY: 500 #ms leaveProject: (io, client, callback = (error) ->) -> - metrics.inc "editor.leave-project" Utils.getClientAttributes client, ["project_id", "user_id"], (error, {project_id, user_id}) -> return callback(error) if error? + return callback() unless project_id # client did not join project + metrics.inc "editor.leave-project" logger.log {project_id, user_id, client_id: client.id}, "client leaving project" WebsocketLoadBalancer.emitToRoom project_id, "clientTracking.clientDisconnected", client.id - # bail out if the client had not managed to authenticate or join - # the project. Prevents downstream errors in docupdater from - # flushProjectToMongoAndDelete with null project_id. - if not user_id? - logger.log {client_id: client.id}, "client leaving, unknown user" - return callback() - if not project_id? - logger.log {user_id: user_id, client_id: client.id}, "client leaving, not in project" - return callback() - # We can do this in the background ConnectedUsersManager.markUserAsDisconnected project_id, client.id, (err) -> if err? diff --git a/services/real-time/test/unit/coffee/WebsocketControllerTests.coffee b/services/real-time/test/unit/coffee/WebsocketControllerTests.coffee index 62243f797c..ac5e9f76fd 100644 --- a/services/real-time/test/unit/coffee/WebsocketControllerTests.coffee +++ b/services/real-time/test/unit/coffee/WebsocketControllerTests.coffee @@ -143,6 +143,19 @@ describe 'WebsocketController', -> @WebsocketController.FLUSH_IF_EMPTY_DELAY = 0 tk.reset() # Allow setTimeout to work. + describe "when the client did not joined a project yet", -> + beforeEach (done) -> + @client.params = {} + @WebsocketController.leaveProject @io, @client, done + + it "should bail out when calling leaveProject", () -> + @WebsocketLoadBalancer.emitToRoom.called.should.equal false + @RoomManager.leaveProjectAndDocs.called.should.equal false + @ConnectedUsersManager.markUserAsDisconnected.called.should.equal false + + it "should not inc any metric", () -> + @metrics.inc.called.should.equal false + describe "when the project is empty", -> beforeEach (done) -> @clientsInRoom = [] @@ -201,8 +214,8 @@ describe 'WebsocketController', -> .calledWith(@project_id) .should.equal false - it "should increment the leave-project metric", -> - @metrics.inc.calledWith("editor.leave-project").should.equal true + it "should not increment the leave-project metric", -> + @metrics.inc.calledWith("editor.leave-project").should.equal false describe "when client has not joined a project", -> beforeEach (done) -> @@ -225,8 +238,8 @@ describe 'WebsocketController', -> .calledWith(@project_id) .should.equal false - it "should increment the leave-project metric", -> - @metrics.inc.calledWith("editor.leave-project").should.equal true + it "should not increment the leave-project metric", -> + @metrics.inc.calledWith("editor.leave-project").should.equal false describe "joinDoc", -> beforeEach -> From 17d04b9041a582e06189705001680df337ee5f25 Mon Sep 17 00:00:00 2001 From: Jakob Ackermann Date: Thu, 30 Apr 2020 18:46:21 +0200 Subject: [PATCH 332/491] [misc] bump sinon to 2.x for promise support with stubs (cherry picked from commit 3c92b937f9430175d7c51660e03c507635448e88) --- services/real-time/package-lock.json | 215 ++++++++------------------- services/real-time/package.json | 2 +- 2 files changed, 63 insertions(+), 154 deletions(-) diff --git a/services/real-time/package-lock.json b/services/real-time/package-lock.json index 2fa57937eb..1541d23171 100644 --- a/services/real-time/package-lock.json +++ b/services/real-time/package-lock.json @@ -666,15 +666,6 @@ "type-detect": "0.1.1" } }, - "define-properties": { - "version": "1.1.3", - "resolved": "https://registry.npmjs.org/define-properties/-/define-properties-1.1.3.tgz", - "integrity": "sha1-z4jabL7ib+bbcJT2HYcMvYTO6fE=", - "dev": true, - "requires": { - "object-keys": "^1.0.12" - } - }, "delay": { "version": "4.3.0", "resolved": "https://registry.npmjs.org/delay/-/delay-4.3.0.tgz", @@ -700,6 +691,12 @@ "resolved": "https://registry.npmjs.org/destroy/-/destroy-1.0.4.tgz", "integrity": "sha1-l4hXRCxEdJ5CBmE+N5RiBYJqvYA=" }, + "diff": { + "version": "3.5.0", + "resolved": "https://registry.npmjs.org/diff/-/diff-3.5.0.tgz", + "integrity": "sha512-A46qtFgd+g7pDZinpnwiRJtxbC1hpgf0uzP3iG89scHk0AUC7A1TGxf5OiiOUv/JMZR8GOt8hL900hV0bOy5xA==", + "dev": true + }, "dtrace-provider": { "version": "0.2.8", "resolved": "https://registry.npmjs.org/dtrace-provider/-/dtrace-provider-0.2.8.tgz", @@ -766,31 +763,6 @@ "resolved": "https://registry.npmjs.org/ent/-/ent-2.2.0.tgz", "integrity": "sha1-6WQhkyWiHQX0RGai9obtbOX13R0=" }, - "es-abstract": { - "version": "1.13.0", - "resolved": "https://registry.npmjs.org/es-abstract/-/es-abstract-1.13.0.tgz", - "integrity": "sha1-rIYUX91QmdjdSVWMy6Lq+biOJOk=", - "dev": true, - "requires": { - "es-to-primitive": "^1.2.0", - "function-bind": "^1.1.1", - "has": "^1.0.3", - "is-callable": "^1.1.4", - "is-regex": "^1.0.4", - "object-keys": "^1.0.12" - } - }, - "es-to-primitive": { - "version": "1.2.0", - "resolved": "https://registry.npmjs.org/es-to-primitive/-/es-to-primitive-1.2.0.tgz", - "integrity": "sha1-7fckeAM0VujdqO8J4ArZZQcH83c=", - "dev": true, - "requires": { - "is-callable": "^1.1.4", - "is-date-object": "^1.0.1", - "is-symbol": "^1.0.2" - } - }, "es6-promise": { "version": "4.2.8", "resolved": "https://registry.npmjs.org/es6-promise/-/es6-promise-4.2.8.tgz", @@ -1046,12 +1018,12 @@ } }, "formatio": { - "version": "1.1.1", - "resolved": "https://registry.npmjs.org/formatio/-/formatio-1.1.1.tgz", - "integrity": "sha1-XtPM1jZVEJc4NGXZlhmRAOhhYek=", + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/formatio/-/formatio-1.2.0.tgz", + "integrity": "sha1-87IWfZBoxGmKjVH092CjmlTYGOs=", "dev": true, "requires": { - "samsam": "~1.1" + "samsam": "1.x" } }, "forwarded": { @@ -1070,12 +1042,6 @@ "integrity": "sha1-FQStJSMVjKpA20onh8sBQRmU6k8=", "dev": true }, - "function-bind": { - "version": "1.1.1", - "resolved": "https://registry.npmjs.org/function-bind/-/function-bind-1.1.1.tgz", - "integrity": "sha1-pWiZ0+o8m6uHS7l3O3xe3pL0iV0=", - "dev": true - }, "gaxios": { "version": "1.8.4", "resolved": "https://registry.npmjs.org/gaxios/-/gaxios-1.8.4.tgz", @@ -1189,21 +1155,6 @@ "har-schema": "^2.0.0" } }, - "has": { - "version": "1.0.3", - "resolved": "https://registry.npmjs.org/has/-/has-1.0.3.tgz", - "integrity": "sha1-ci18v8H2qoJB8W3YFOAR4fQeh5Y=", - "dev": true, - "requires": { - "function-bind": "^1.1.1" - } - }, - "has-symbols": { - "version": "1.0.0", - "resolved": "https://registry.npmjs.org/has-symbols/-/has-symbols-1.0.0.tgz", - "integrity": "sha1-uhqPGvKg/DllD1yFA2dwQSIGO0Q=", - "dev": true - }, "he": { "version": "1.1.1", "resolved": "https://registry.npmjs.org/he/-/he-1.1.1.tgz", @@ -1323,53 +1274,11 @@ "resolved": "https://registry.npmjs.org/is/-/is-3.3.0.tgz", "integrity": "sha512-nW24QBoPcFGGHJGUwnfpI7Yc5CdqWNdsyHQszVE/z2pKHXzh7FZ5GWhJqSyaQ9wMkQnsTx+kAI8bHlCX4tKdbg==" }, - "is-arguments": { - "version": "1.0.4", - "resolved": "https://registry.npmjs.org/is-arguments/-/is-arguments-1.0.4.tgz", - "integrity": "sha1-P6+WbHy6D/Q3+zH2JQCC/PBEjPM=", - "dev": true - }, "is-buffer": { "version": "2.0.4", "resolved": "https://registry.npmjs.org/is-buffer/-/is-buffer-2.0.4.tgz", "integrity": "sha512-Kq1rokWXOPXWuaMAqZiJW4XxsmD9zGx9q4aePabbn3qCRGedtH7Cm+zV8WETitMfu1wdh+Rvd6w5egwSngUX2A==" }, - "is-callable": { - "version": "1.1.4", - "resolved": "https://registry.npmjs.org/is-callable/-/is-callable-1.1.4.tgz", - "integrity": "sha1-HhrfIZ4e62hNaR+dagX/DTCiTXU=", - "dev": true - }, - "is-date-object": { - "version": "1.0.1", - "resolved": "https://registry.npmjs.org/is-date-object/-/is-date-object-1.0.1.tgz", - "integrity": "sha1-mqIOtq7rv/d/vTPnTKAbM1gdOhY=", - "dev": true - }, - "is-generator-function": { - "version": "1.0.7", - "resolved": "https://registry.npmjs.org/is-generator-function/-/is-generator-function-1.0.7.tgz", - "integrity": "sha1-0hMuUpuwAAp/gHlNS99c1eWBNSI=", - "dev": true - }, - "is-regex": { - "version": "1.0.4", - "resolved": "https://registry.npmjs.org/is-regex/-/is-regex-1.0.4.tgz", - "integrity": "sha1-VRdIm1RwkbCTDglWVM7SXul+lJE=", - "dev": true, - "requires": { - "has": "^1.0.1" - } - }, - "is-symbol": { - "version": "1.0.2", - "resolved": "https://registry.npmjs.org/is-symbol/-/is-symbol-1.0.2.tgz", - "integrity": "sha1-oFX2rlcZLK7jKeeoYBGLSXqVDzg=", - "dev": true, - "requires": { - "has-symbols": "^1.0.0" - } - }, "is-typedarray": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/is-typedarray/-/is-typedarray-1.0.0.tgz", @@ -1503,9 +1412,9 @@ } }, "lolex": { - "version": "1.3.2", - "resolved": "https://registry.npmjs.org/lolex/-/lolex-1.3.2.tgz", - "integrity": "sha1-fD2mL/yzDw9agKJWbKJORdigHzE=", + "version": "1.6.0", + "resolved": "https://registry.npmjs.org/lolex/-/lolex-1.6.0.tgz", + "integrity": "sha1-OpoCg0UqR9dDnnJzG54H1zhuSfY=", "dev": true }, "long": { @@ -1730,6 +1639,12 @@ "integrity": "sha1-OSHhECMtHreQ89rGG7NwUxx9NW4=", "dev": true }, + "native-promise-only": { + "version": "0.8.1", + "resolved": "https://registry.npmjs.org/native-promise-only/-/native-promise-only-0.8.1.tgz", + "integrity": "sha1-IKMYwwy0X3H+et+/eyHJnBRy7xE=", + "dev": true + }, "ncp": { "version": "2.0.0", "resolved": "https://registry.npmjs.org/ncp/-/ncp-2.0.0.tgz", @@ -1756,24 +1671,6 @@ "resolved": "https://registry.npmjs.org/oauth-sign/-/oauth-sign-0.9.0.tgz", "integrity": "sha1-R6ewFrqmi1+g7PPe4IqFxnmsZFU=" }, - "object-keys": { - "version": "1.1.1", - "resolved": "https://registry.npmjs.org/object-keys/-/object-keys-1.1.1.tgz", - "integrity": "sha1-HEfyct8nfzsdrwYWd9nILiMixg4=", - "dev": true - }, - "object.entries": { - "version": "1.1.0", - "resolved": "https://registry.npmjs.org/object.entries/-/object.entries-1.1.0.tgz", - "integrity": "sha1-ICT8bWuiRq7ji9sP/Vz7zzcbdRk=", - "dev": true, - "requires": { - "define-properties": "^1.1.3", - "es-abstract": "^1.12.0", - "function-bind": "^1.1.1", - "has": "^1.0.3" - } - }, "on-finished": { "version": "2.3.0", "resolved": "https://registry.npmjs.org/on-finished/-/on-finished-2.3.0.tgz", @@ -2202,9 +2099,9 @@ "integrity": "sha1-RPoWGwGHuVSd2Eu5GAL5vYOFzWo=" }, "samsam": { - "version": "1.1.2", - "resolved": "https://registry.npmjs.org/samsam/-/samsam-1.1.2.tgz", - "integrity": "sha1-vsEf3IOp/aBjQBIQ5AF2wwJNFWc=", + "version": "1.3.0", + "resolved": "https://registry.npmjs.org/samsam/-/samsam-1.3.0.tgz", + "integrity": "sha512-1HwIYD/8UlOtFS3QO3w7ey+SdSDFE4HRNLZoZRYVQefrOY3l17epswImeB1ijgJFQJodIaHcwkp3r/myBjFVbg==", "dev": true }, "sandboxed-module": { @@ -2299,15 +2196,42 @@ "integrity": "sha512-sQTKC1Re/rM6XyFM6fIAGHRPVGvyXfgzIDvzoq608vM+jeyVD0Tu1E6Np0Kc2zAIFWIj963V2800iF/9LPieQw==" }, "sinon": { - "version": "1.17.7", - "resolved": "https://registry.npmjs.org/sinon/-/sinon-1.17.7.tgz", - "integrity": "sha1-RUKk9JugxFwF6y6d2dID4rjv4L8=", + "version": "2.4.1", + "resolved": "https://registry.npmjs.org/sinon/-/sinon-2.4.1.tgz", + "integrity": "sha512-vFTrO9Wt0ECffDYIPSP/E5bBugt0UjcBQOfQUMh66xzkyPEnhl/vM2LRZi2ajuTdkH07sA6DzrM6KvdvGIH8xw==", "dev": true, "requires": { - "formatio": "1.1.1", - "lolex": "1.3.2", - "samsam": "1.1.2", - "util": ">=0.10.3 <1" + "diff": "^3.1.0", + "formatio": "1.2.0", + "lolex": "^1.6.0", + "native-promise-only": "^0.8.1", + "path-to-regexp": "^1.7.0", + "samsam": "^1.1.3", + "text-encoding": "0.6.4", + "type-detect": "^4.0.0" + }, + "dependencies": { + "isarray": { + "version": "0.0.1", + "resolved": "https://registry.npmjs.org/isarray/-/isarray-0.0.1.tgz", + "integrity": "sha1-ihis/Kmo9Bd+Cav8YDiTmwXR7t8=", + "dev": true + }, + "path-to-regexp": { + "version": "1.8.0", + "resolved": "https://registry.npmjs.org/path-to-regexp/-/path-to-regexp-1.8.0.tgz", + "integrity": "sha512-n43JRhlUKUAlibEJhPeir1ncUID16QnEjNpwzNdO3Lm4ywrBpBZ5oLD0I6br9evr1Y9JTqwRtAh7JLoOzAQdVA==", + "dev": true, + "requires": { + "isarray": "0.0.1" + } + }, + "type-detect": { + "version": "4.0.8", + "resolved": "https://registry.npmjs.org/type-detect/-/type-detect-4.0.8.tgz", + "integrity": "sha512-0fr/mIH1dlO+x7TlcMy+bIDqKPsw/70tVyeHW787goQjhmqaZe10uwLujubK9q9Lg6Fiho1KUKDYz0Z7k7g5/g==", + "dev": true + } } }, "socket.io": { @@ -2453,6 +2377,12 @@ } } }, + "text-encoding": { + "version": "0.6.4", + "resolved": "https://registry.npmjs.org/text-encoding/-/text-encoding-0.6.4.tgz", + "integrity": "sha1-45mpgiV6J22uQou5KEXLcb3CbRk=", + "dev": true + }, "through": { "version": "2.3.8", "resolved": "https://registry.npmjs.org/through/-/through-2.3.8.tgz", @@ -2554,27 +2484,6 @@ } } }, - "util": { - "version": "0.12.1", - "resolved": "https://registry.npmjs.org/util/-/util-0.12.1.tgz", - "integrity": "sha1-+QjntjPnOWx2TmlN0U5xYlbOit4=", - "dev": true, - "requires": { - "inherits": "^2.0.3", - "is-arguments": "^1.0.4", - "is-generator-function": "^1.0.7", - "object.entries": "^1.1.0", - "safe-buffer": "^5.1.2" - }, - "dependencies": { - "safe-buffer": { - "version": "5.2.0", - "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.2.0.tgz", - "integrity": "sha1-t02uxJsRSPiMZLaNSbHoFcHy9Rk=", - "dev": true - } - } - }, "util-deprecate": { "version": "1.0.2", "resolved": "https://registry.npmjs.org/util-deprecate/-/util-deprecate-1.0.2.tgz", diff --git a/services/real-time/package.json b/services/real-time/package.json index ea3db675ce..1e20ac28fc 100644 --- a/services/real-time/package.json +++ b/services/real-time/package.json @@ -42,7 +42,7 @@ "chai": "~1.9.1", "cookie-signature": "^1.0.5", "sandboxed-module": "~0.3.0", - "sinon": "^1.5.2", + "sinon": "^2.4.1", "mocha": "^4.0.1", "uid-safe": "^1.0.1", "timekeeper": "0.0.4" From 1095851dfe00cc725665fb44e1c8c84810c8240f Mon Sep 17 00:00:00 2001 From: Jakob Ackermann Date: Fri, 1 May 2020 12:43:18 +0200 Subject: [PATCH 333/491] [misc] test/unit: fix typos and assertion of error messages Sinon does not check the contents of the passed error when checked via sinon.stub().calledWith. ``` callback = sinon.stub() callback(new Error("some message")) .calledWith(new Error("completely different message")) === true ``` Cherry-pick plus an additional patch for the joinProject bail-out. (cherry picked from commit d9570fee70701a5f431c39fdbec5f8bc5a7843fe) --- .../coffee/AuthorizationManagerTests.coffee | 60 +++++++------------ .../unit/coffee/WebApiManagerTests.coffee | 6 +- .../coffee/WebsocketControllerTests.coffee | 6 +- 3 files changed, 28 insertions(+), 44 deletions(-) diff --git a/services/real-time/test/unit/coffee/AuthorizationManagerTests.coffee b/services/real-time/test/unit/coffee/AuthorizationManagerTests.coffee index 9856684247..46b9e8be9a 100644 --- a/services/real-time/test/unit/coffee/AuthorizationManagerTests.coffee +++ b/services/real-time/test/unit/coffee/AuthorizationManagerTests.coffee @@ -27,19 +27,19 @@ describe 'AuthorizationManager', -> @AuthorizationManager.assertClientCanViewProject @client, (error) -> expect(error).to.be.null done() - + it "should allow the readAndWrite privilegeLevel", (done) -> @client.params.privilege_level = "readAndWrite" @AuthorizationManager.assertClientCanViewProject @client, (error) -> expect(error).to.be.null done() - + it "should allow the owner privilegeLevel", (done) -> @client.params.privilege_level = "owner" @AuthorizationManager.assertClientCanViewProject @client, (error) -> expect(error).to.be.null done() - + it "should return an error with any other privilegeLevel", (done) -> @client.params.privilege_level = "unknown" @AuthorizationManager.assertClientCanViewProject @client, (error) -> @@ -52,19 +52,19 @@ describe 'AuthorizationManager', -> @AuthorizationManager.assertClientCanEditProject @client, (error) -> error.message.should.equal "not authorized" done() - + it "should allow the readAndWrite privilegeLevel", (done) -> @client.params.privilege_level = "readAndWrite" @AuthorizationManager.assertClientCanEditProject @client, (error) -> expect(error).to.be.null done() - + it "should allow the owner privilegeLevel", (done) -> @client.params.privilege_level = "owner" @AuthorizationManager.assertClientCanEditProject @client, (error) -> expect(error).to.be.null done() - + it "should return an error with any other privilegeLevel", (done) -> @client.params.privilege_level = "unknown" @AuthorizationManager.assertClientCanEditProject @client, (error) -> @@ -84,20 +84,16 @@ describe 'AuthorizationManager', -> @client.params.privilege_level = "unknown" it "should not allow access", () -> - @AuthorizationManager.assertClientCanViewProjectAndDoc @client, @doc_id, @callback - @callback - .calledWith(new Error("not authorised")) - .should.equal true + @AuthorizationManager.assertClientCanViewProjectAndDoc @client, @doc_id, (err) -> + err.message.should.equal "not authorized" describe "even when authorised at the doc level", -> beforeEach (done) -> @AuthorizationManager.addAccessToDoc @client, @doc_id, done it "should not allow access", () -> - @AuthorizationManager.assertClientCanViewProjectAndDoc @client, @doc_id, @callback - @callback - .calledWith(new Error("not authorised")) - .should.equal true + @AuthorizationManager.assertClientCanViewProjectAndDoc @client, @doc_id, (err) -> + err.message.should.equal "not authorized" describe "when authorised at the project level", -> beforeEach () -> @@ -105,10 +101,8 @@ describe 'AuthorizationManager', -> describe "and not authorised at the document level", -> it "should not allow access", () -> - @AuthorizationManager.assertClientCanViewProjectAndDoc @client, @doc_id, @callback - @callback - .calledWith(new Error("not authorised")) - .should.equal true + @AuthorizationManager.assertClientCanViewProjectAndDoc @client, @doc_id, (err) -> + err.message.should.equal "not authorized" describe "and authorised at the document level", -> beforeEach (done) -> @@ -126,10 +120,8 @@ describe 'AuthorizationManager', -> @AuthorizationManager.removeAccessToDoc @client, @doc_id, done it "should deny access", () -> - @AuthorizationManager.assertClientCanViewProjectAndDoc @client, @doc_id, @callback - @callback - .calledWith(new Error("not authorised")) - .should.equal true + @AuthorizationManager.assertClientCanViewProjectAndDoc @client, @doc_id, (err) -> + err.message.should.equal "not authorized" describe "assertClientCanEditProjectAndDoc", -> beforeEach () -> @@ -142,20 +134,16 @@ describe 'AuthorizationManager', -> @client.params.privilege_level = "readOnly" it "should not allow access", () -> - @AuthorizationManager.assertClientCanEditProjectAndDoc @client, @doc_id, @callback - @callback - .calledWith(new Error("not authorised")) - .should.equal true + @AuthorizationManager.assertClientCanEditProjectAndDoc @client, @doc_id, (err) -> + err.message.should.equal "not authorized" describe "even when authorised at the doc level", -> beforeEach (done) -> @AuthorizationManager.addAccessToDoc @client, @doc_id, done it "should not allow access", () -> - @AuthorizationManager.assertClientCanEditProjectAndDoc @client, @doc_id, @callback - @callback - .calledWith(new Error("not authorised")) - .should.equal true + @AuthorizationManager.assertClientCanEditProjectAndDoc @client, @doc_id, (err) -> + err.message.should.equal "not authorized" describe "when authorised at the project level", -> beforeEach () -> @@ -163,10 +151,8 @@ describe 'AuthorizationManager', -> describe "and not authorised at the document level", -> it "should not allow access", () -> - @AuthorizationManager.assertClientCanEditProjectAndDoc @client, @doc_id, @callback - @callback - .calledWith(new Error("not authorised")) - .should.equal true + @AuthorizationManager.assertClientCanEditProjectAndDoc @client, @doc_id, (err) -> + err.message.should.equal "not authorized" describe "and authorised at the document level", -> beforeEach (done) -> @@ -184,7 +170,5 @@ describe 'AuthorizationManager', -> @AuthorizationManager.removeAccessToDoc @client, @doc_id, done it "should deny access", () -> - @AuthorizationManager.assertClientCanEditProjectAndDoc @client, @doc_id, @callback - @callback - .calledWith(new Error("not authorised")) - .should.equal true + @AuthorizationManager.assertClientCanEditProjectAndDoc @client, @doc_id, (err) -> + err.message.should.equal "not authorized" diff --git a/services/real-time/test/unit/coffee/WebApiManagerTests.coffee b/services/real-time/test/unit/coffee/WebApiManagerTests.coffee index a87522c0a9..e65ba93859 100644 --- a/services/real-time/test/unit/coffee/WebApiManagerTests.coffee +++ b/services/real-time/test/unit/coffee/WebApiManagerTests.coffee @@ -60,7 +60,7 @@ describe 'WebApiManager', -> it "should call the callback with an error", -> @callback - .calledWith(new Error("non-success code from web: 500")) + .calledWith(sinon.match({message: "non-success status code from web: 500"})) .should.equal true describe "with no data from web", -> @@ -70,7 +70,7 @@ describe 'WebApiManager', -> it "should call the callback with an error", -> @callback - .calledWith(new Error("no data returned from joinProject request")) + .calledWith(sinon.match({message: "no data returned from joinProject request"})) .should.equal true describe "when the project is over its rate limit", -> @@ -80,5 +80,5 @@ describe 'WebApiManager', -> it "should call the callback with a TooManyRequests error code", -> @callback - .calledWith(new CodedError("rate-limit hit when joining project", "TooManyRequests")) + .calledWith(sinon.match({message: "rate-limit hit when joining project", code: "TooManyRequests"})) .should.equal true diff --git a/services/real-time/test/unit/coffee/WebsocketControllerTests.coffee b/services/real-time/test/unit/coffee/WebsocketControllerTests.coffee index 79b22dbc81..498b425281 100644 --- a/services/real-time/test/unit/coffee/WebsocketControllerTests.coffee +++ b/services/real-time/test/unit/coffee/WebsocketControllerTests.coffee @@ -119,7 +119,7 @@ describe 'WebsocketController', -> it "should return an error", -> @callback - .calledWith(new Error("not authorized")) + .calledWith(sinon.match({message: "not authorized"})) .should.equal true it "should not log an error", -> @@ -143,7 +143,7 @@ describe 'WebsocketController', -> it "should return an error", -> @callback - .calledWith(new Error("subscribe failed")) + .calledWith(sinon.match({message: "subscribe failed"})) .should.equal true @callback.args[0][0].message.should.equal "subscribe failed" @@ -369,7 +369,7 @@ describe 'WebsocketController', -> @WebsocketController.joinDoc @client, @doc_id, -1, @options, @callback it "should call the callback with an error", -> - @callback.calledWith(@err).should.equal true + @callback.calledWith(sinon.match({message: "not authorized"})).should.equal true it "should not call the DocumentUpdaterManager", -> @DocumentUpdaterManager.getDocument.called.should.equal false From 41debfae0fee378fae7ecbcb53d22f157ad8d241 Mon Sep 17 00:00:00 2001 From: Jakob Ackermann Date: Fri, 15 May 2020 11:34:07 +0200 Subject: [PATCH 334/491] [ChannelManager] rework (un)subscribing to redis - send a subscribe request on every request - wait for a pending unsubscribe request before subscribing - wait for a pending subscribe request before unsubscribing Co-Authored-By: Brian Gough --- .../app/coffee/ChannelManager.coffee | 63 +++++++++++-------- 1 file changed, 38 insertions(+), 25 deletions(-) diff --git a/services/real-time/app/coffee/ChannelManager.coffee b/services/real-time/app/coffee/ChannelManager.coffee index 367d2059a2..e60a145bd5 100644 --- a/services/real-time/app/coffee/ChannelManager.coffee +++ b/services/real-time/app/coffee/ChannelManager.coffee @@ -17,35 +17,48 @@ module.exports = ChannelManager = subscribe: (rclient, baseChannel, id) -> clientChannelMap = @getClientMapEntry(rclient) channel = "#{baseChannel}:#{id}" - # we track pending subscribes because we want to be sure that the - # channel is active before letting the client join the doc or project, - # so that events are not lost. - if clientChannelMap.has(channel) - logger.warn {channel}, "subscribe already actioned" - # return the existing subscribe promise, so we can wait for it to resolve - return clientChannelMap.get(channel) - else - # get the subscribe promise and return it, the actual subscribe - # completes in the background - subscribePromise = rclient.subscribe channel - clientChannelMap.set(channel, subscribePromise) - logger.log {channel}, "subscribed to new channel" - metrics.inc "subscribe.#{baseChannel}" - return subscribePromise + actualSubscribe = () -> + # subscribe is happening in the foreground and it should reject + p = rclient.subscribe(channel) + p.finally () -> + if clientChannelMap.get(channel) is subscribePromise + clientChannelMap.delete(channel) + .then () -> + logger.log {channel}, "subscribed to channel" + metrics.inc "subscribe.#{baseChannel}" + .catch (err) -> + logger.error {channel, err}, "failed to subscribe to channel" + metrics.inc "subscribe.failed.#{baseChannel}" + return p + + pendingActions = clientChannelMap.get(channel) || Promise.resolve() + subscribePromise = pendingActions.then(actualSubscribe, actualSubscribe) + clientChannelMap.set(channel, subscribePromise) + logger.log {channel}, "planned to subscribe to channel" + return subscribePromise unsubscribe: (rclient, baseChannel, id) -> clientChannelMap = @getClientMapEntry(rclient) channel = "#{baseChannel}:#{id}" - # we don't need to track pending unsubscribes, because we there is no - # harm if events continue to arrive on the channel while the unsubscribe - # command in pending. - if !clientChannelMap.has(channel) - logger.error {channel}, "not subscribed - shouldn't happen" - else - rclient.unsubscribe channel # completes in the background - clientChannelMap.delete(channel) - logger.log {channel}, "unsubscribed from channel" - metrics.inc "unsubscribe.#{baseChannel}" + actualUnsubscribe = () -> + # unsubscribe is happening in the background, it should not reject + p = rclient.unsubscribe(channel) + .finally () -> + if clientChannelMap.get(channel) is unsubscribePromise + clientChannelMap.delete(channel) + .then () -> + logger.log {channel}, "unsubscribed from channel" + metrics.inc "unsubscribe.#{baseChannel}" + .catch (err) -> + logger.error {channel, err}, "unsubscribed from channel" + metrics.inc "unsubscribe.failed.#{baseChannel}" + return p + + pendingActions = clientChannelMap.get(channel) || Promise.resolve() + unsubscribePromise = pendingActions.then(actualUnsubscribe, actualUnsubscribe) + clientChannelMap.set(channel, unsubscribePromise) + logger.log {channel}, "planned to unsubscribe from channel" + return unsubscribePromise publish: (rclient, baseChannel, id, data) -> metrics.summary "redis.publish.#{baseChannel}", data.length From d13acb8ca306447754a44e2fe58d641a90aa0f27 Mon Sep 17 00:00:00 2001 From: Jakob Ackermann Date: Fri, 15 May 2020 11:43:11 +0200 Subject: [PATCH 335/491] [ChannelManager] port related and still mostly valid test from v2 I skipped the bulk of verifyConsistentBehaviour tests which are not valid for the new implementation -- there is no optimization and always cleanup. --- .../coffee/LeaveProjectTests.coffee | 77 +++++-- .../test/acceptance/coffee/PubSubRace.coffee | 205 ++++++++++++++++++ .../unit/coffee/ChannelManagerTests.coffee | 149 +++++++++++-- 3 files changed, 392 insertions(+), 39 deletions(-) create mode 100644 services/real-time/test/acceptance/coffee/PubSubRace.coffee diff --git a/services/real-time/test/acceptance/coffee/LeaveProjectTests.coffee b/services/real-time/test/acceptance/coffee/LeaveProjectTests.coffee index 7ef8b35c64..5925472245 100644 --- a/services/real-time/test/acceptance/coffee/LeaveProjectTests.coffee +++ b/services/real-time/test/acceptance/coffee/LeaveProjectTests.coffee @@ -4,10 +4,14 @@ FixturesManager = require "./helpers/FixturesManager" async = require "async" +settings = require "settings-sharelatex" +redis = require "redis-sharelatex" +rclient = redis.createClient(settings.redis.pubsub) + describe "leaveProject", -> before (done) -> MockDocUpdaterServer.run done - + describe "with other clients in the project", -> before (done) -> async.series [ @@ -18,53 +22,76 @@ describe "leaveProject", -> name: "Test Project" } }, (e, {@project_id, @user_id}) => cb() - + (cb) => @clientA = RealTimeClient.connect() @clientA.on "connectionAccepted", cb - + (cb) => @clientB = RealTimeClient.connect() @clientB.on "connectionAccepted", cb - + @clientBDisconnectMessages = [] @clientB.on "clientTracking.clientDisconnected", (data) => @clientBDisconnectMessages.push data - + (cb) => @clientA.emit "joinProject", project_id: @project_id, (error, @project, @privilegeLevel, @protocolVersion) => cb(error) - + (cb) => @clientB.emit "joinProject", project_id: @project_id, (error, @project, @privilegeLevel, @protocolVersion) => cb(error) - + + (cb) => + FixturesManager.setUpDoc @project_id, {@lines, @version, @ops}, (e, {@doc_id}) => + cb(e) + + (cb) => + @clientA.emit "joinDoc", @doc_id, cb + (cb) => + @clientB.emit "joinDoc", @doc_id, cb + (cb) => # leaveProject is called when the client disconnects @clientA.on "disconnect", () -> cb() @clientA.disconnect() - + (cb) => # The API waits a little while before flushing changes setTimeout done, 1000 - + ], done it "should emit a disconnect message to the room", -> @clientBDisconnectMessages.should.deep.equal [@clientA.socket.sessionid] - + it "should no longer list the client in connected users", (done) -> @clientB.emit "clientTracking.getConnectedUsers", (error, users) => for user in users if user.client_id == @clientA.socket.sessionid throw "Expected clientA to not be listed in connected users" return done() - + it "should not flush the project to the document updater", -> MockDocUpdaterServer.deleteProject .calledWith(@project_id) .should.equal false + it "should remain subscribed to the editor-events channels", (done) -> + rclient.pubsub 'CHANNELS', (err, resp) => + return done(err) if err + resp.should.include "editor-events:#{@project_id}" + done() + return null + + it "should remain subscribed to the applied-ops channels", (done) -> + rclient.pubsub 'CHANNELS', (err, resp) => + return done(err) if err + resp.should.include "applied-ops:#{@doc_id}" + done() + return null + describe "with no other clients in the project", -> before (done) -> async.series [ @@ -75,20 +102,26 @@ describe "leaveProject", -> name: "Test Project" } }, (e, {@project_id, @user_id}) => cb() - + (cb) => @clientA = RealTimeClient.connect() @clientA.on "connect", cb - + (cb) => @clientA.emit "joinProject", project_id: @project_id, (error, @project, @privilegeLevel, @protocolVersion) => cb(error) - + + (cb) => + FixturesManager.setUpDoc @project_id, {@lines, @version, @ops}, (e, {@doc_id}) => + cb(e) + (cb) => + @clientA.emit "joinDoc", @doc_id, cb + (cb) => # leaveProject is called when the client disconnects @clientA.on "disconnect", () -> cb() @clientA.disconnect() - + (cb) => # The API waits a little while before flushing changes setTimeout done, 1000 @@ -98,3 +131,17 @@ describe "leaveProject", -> MockDocUpdaterServer.deleteProject .calledWith(@project_id) .should.equal true + + it "should not subscribe to the editor-events channels anymore", (done) -> + rclient.pubsub 'CHANNELS', (err, resp) => + return done(err) if err + resp.should.not.include "editor-events:#{@project_id}" + done() + return null + + it "should not subscribe to the applied-ops channels anymore", (done) -> + rclient.pubsub 'CHANNELS', (err, resp) => + return done(err) if err + resp.should.not.include "applied-ops:#{@doc_id}" + done() + return null diff --git a/services/real-time/test/acceptance/coffee/PubSubRace.coffee b/services/real-time/test/acceptance/coffee/PubSubRace.coffee new file mode 100644 index 0000000000..d5e6653fac --- /dev/null +++ b/services/real-time/test/acceptance/coffee/PubSubRace.coffee @@ -0,0 +1,205 @@ +RealTimeClient = require "./helpers/RealTimeClient" +MockDocUpdaterServer = require "./helpers/MockDocUpdaterServer" +FixturesManager = require "./helpers/FixturesManager" + +async = require "async" + +settings = require "settings-sharelatex" +redis = require "redis-sharelatex" +rclient = redis.createClient(settings.redis.pubsub) + +describe "PubSubRace", -> + before (done) -> + MockDocUpdaterServer.run done + + describe "when the client leaves a doc before joinDoc completes", -> + before (done) -> + async.series [ + (cb) => + FixturesManager.setUpProject { + privilegeLevel: "owner" + project: { + name: "Test Project" + } + }, (e, {@project_id, @user_id}) => cb() + + (cb) => + @clientA = RealTimeClient.connect() + @clientA.on "connect", cb + + (cb) => + @clientA.emit "joinProject", project_id: @project_id, (error, @project, @privilegeLevel, @protocolVersion) => + cb(error) + + (cb) => + FixturesManager.setUpDoc @project_id, {@lines, @version, @ops}, (e, {@doc_id}) => + cb(e) + + (cb) => + @clientA.emit "joinDoc", @doc_id, () -> + # leave before joinDoc completes + @clientA.emit "leaveDoc", @doc_id, cb + + (cb) => + # wait for subscribe and unsubscribe + setTimeout cb, 100 + ], done + + it "should not subscribe to the applied-ops channels anymore", (done) -> + rclient.pubsub 'CHANNELS', (err, resp) => + return done(err) if err + resp.should.not.include "applied-ops:#{@doc_id}" + done() + return null + + describe "when the client emits joinDoc and leaveDoc requests frequently and leaves eventually", -> + before (done) -> + async.series [ + (cb) => + FixturesManager.setUpProject { + privilegeLevel: "owner" + project: { + name: "Test Project" + } + }, (e, {@project_id, @user_id}) => cb() + + (cb) => + @clientA = RealTimeClient.connect() + @clientA.on "connect", cb + + (cb) => + @clientA.emit "joinProject", project_id: @project_id, (error, @project, @privilegeLevel, @protocolVersion) => + cb(error) + + (cb) => + FixturesManager.setUpDoc @project_id, {@lines, @version, @ops}, (e, {@doc_id}) => + cb(e) + + (cb) => + @clientA.emit "joinDoc", @doc_id, () -> + @clientA.emit "leaveDoc", @doc_id, () -> + @clientA.emit "joinDoc", @doc_id, () -> + @clientA.emit "leaveDoc", @doc_id, () -> + @clientA.emit "joinDoc", @doc_id, () -> + @clientA.emit "leaveDoc", @doc_id, () -> + @clientA.emit "joinDoc", @doc_id, () -> + @clientA.emit "leaveDoc", @doc_id, () -> + @clientA.emit "joinDoc", @doc_id, () -> + @clientA.emit "leaveDoc", @doc_id, cb + + (cb) => + # wait for subscribe and unsubscribe + setTimeout cb, 100 + ], done + + it "should not subscribe to the applied-ops channels anymore", (done) -> + rclient.pubsub 'CHANNELS', (err, resp) => + return done(err) if err + resp.should.not.include "applied-ops:#{@doc_id}" + done() + return null + + describe "when the client emits joinDoc and leaveDoc requests frequently and remains in the doc", -> + before (done) -> + async.series [ + (cb) => + FixturesManager.setUpProject { + privilegeLevel: "owner" + project: { + name: "Test Project" + } + }, (e, {@project_id, @user_id}) => cb() + + (cb) => + @clientA = RealTimeClient.connect() + @clientA.on "connect", cb + + (cb) => + @clientA.emit "joinProject", project_id: @project_id, (error, @project, @privilegeLevel, @protocolVersion) => + cb(error) + + (cb) => + FixturesManager.setUpDoc @project_id, {@lines, @version, @ops}, (e, {@doc_id}) => + cb(e) + + (cb) => + @clientA.emit "joinDoc", @doc_id, () -> + @clientA.emit "leaveDoc", @doc_id, () -> + @clientA.emit "joinDoc", @doc_id, () -> + @clientA.emit "leaveDoc", @doc_id, () -> + @clientA.emit "joinDoc", @doc_id, () -> + @clientA.emit "leaveDoc", @doc_id, () -> + @clientA.emit "joinDoc", @doc_id, () -> + @clientA.emit "leaveDoc", @doc_id, () -> + @clientA.emit "joinDoc", @doc_id, cb + + (cb) => + # wait for subscribe and unsubscribe + setTimeout cb, 100 + ], done + + it "should subscribe to the applied-ops channels", (done) -> + rclient.pubsub 'CHANNELS', (err, resp) => + return done(err) if err + resp.should.include "applied-ops:#{@doc_id}" + done() + return null + + describe "when the client disconnects before joinDoc completes", -> + before (done) -> + async.series [ + (cb) => + FixturesManager.setUpProject { + privilegeLevel: "owner" + project: { + name: "Test Project" + } + }, (e, {@project_id, @user_id}) => cb() + + (cb) => + @clientA = RealTimeClient.connect() + @clientA.on "connect", cb + + (cb) => + @clientA.emit "joinProject", project_id: @project_id, (error, @project, @privilegeLevel, @protocolVersion) => + cb(error) + + (cb) => + FixturesManager.setUpDoc @project_id, {@lines, @version, @ops}, (e, {@doc_id}) => + cb(e) + + (cb) => + joinDocCompleted = false + @clientA.emit "joinDoc", @doc_id, () -> + joinDocCompleted = true + # leave before joinDoc completes + setTimeout () => + if joinDocCompleted + return cb(new Error('joinDocCompleted -- lower timeout')) + @clientA.on "disconnect", () -> cb() + @clientA.disconnect() + # socket.io processes joinDoc and disconnect with different delays: + # - joinDoc goes through two process.nextTick + # - disconnect goes through one process.nextTick + # We have to inject the disconnect event into a different event loop + # cycle. + , 3 + + (cb) => + # wait for subscribe and unsubscribe + setTimeout cb, 100 + ], done + + it "should not subscribe to the editor-events channels anymore", (done) -> + rclient.pubsub 'CHANNELS', (err, resp) => + return done(err) if err + resp.should.not.include "editor-events:#{@project_id}" + done() + return null + + it "should not subscribe to the applied-ops channels anymore", (done) -> + rclient.pubsub 'CHANNELS', (err, resp) => + return done(err) if err + resp.should.not.include "applied-ops:#{@doc_id}" + done() + return null diff --git a/services/real-time/test/unit/coffee/ChannelManagerTests.coffee b/services/real-time/test/unit/coffee/ChannelManagerTests.coffee index edde3e8170..cb77991f58 100644 --- a/services/real-time/test/unit/coffee/ChannelManagerTests.coffee +++ b/services/real-time/test/unit/coffee/ChannelManagerTests.coffee @@ -1,5 +1,6 @@ chai = require('chai') should = chai.should() +expect = chai.expect sinon = require("sinon") modulePath = "../../../app/js/ChannelManager.js" SandboxedModule = require('sandboxed-module') @@ -11,34 +12,82 @@ describe 'ChannelManager', -> @ChannelManager = SandboxedModule.require modulePath, requires: "settings-sharelatex": @settings = {} "metrics-sharelatex": @metrics = {inc: sinon.stub(), summary: sinon.stub()} - "logger-sharelatex": @logger = { log: sinon.stub(), warn: sinon.stub(), error: sinon.stub() } - + describe "subscribe", -> describe "when there is no existing subscription for this redis client", -> - beforeEach -> - @rclient.subscribe = sinon.stub() + beforeEach (done) -> + @rclient.subscribe = sinon.stub().resolves() @ChannelManager.subscribe @rclient, "applied-ops", "1234567890abcdef" + setTimeout done it "should subscribe to the redis channel", -> @rclient.subscribe.calledWithExactly("applied-ops:1234567890abcdef").should.equal true describe "when there is an existing subscription for this redis client", -> - beforeEach -> - @rclient.subscribe = sinon.stub() + beforeEach (done) -> + @rclient.subscribe = sinon.stub().resolves() @ChannelManager.subscribe @rclient, "applied-ops", "1234567890abcdef" - @rclient.subscribe = sinon.stub() # discard the original stub @ChannelManager.subscribe @rclient, "applied-ops", "1234567890abcdef" + setTimeout done - it "should not subscribe to the redis channel", -> - @rclient.subscribe.called.should.equal false + it "should subscribe to the redis channel again", -> + @rclient.subscribe.callCount.should.equal 2 + + describe "when subscribe errors", -> + beforeEach (done) -> + @rclient.subscribe = sinon.stub() + .onFirstCall().rejects(new Error("some redis error")) + .onSecondCall().resolves() + p = @ChannelManager.subscribe @rclient, "applied-ops", "1234567890abcdef" + p.then () -> + done(new Error('should not subscribe but fail')) + .catch (err) => + err.message.should.equal "some redis error" + @ChannelManager.getClientMapEntry(@rclient).has("applied-ops:1234567890abcdef").should.equal false + @ChannelManager.subscribe @rclient, "applied-ops", "1234567890abcdef" + # subscribe is wrapped in Promise, delay other assertions + setTimeout done + return null + + it "should have recorded the error", -> + expect(@metrics.inc.calledWithExactly("subscribe.failed.applied-ops")).to.equal(true) + + it "should subscribe again", -> + @rclient.subscribe.callCount.should.equal 2 + + it "should cleanup", -> + @ChannelManager.getClientMapEntry(@rclient).has("applied-ops:1234567890abcdef").should.equal false + + describe "when subscribe errors and the clientChannelMap entry was replaced", -> + beforeEach (done) -> + @rclient.subscribe = sinon.stub() + .onFirstCall().rejects(new Error("some redis error")) + .onSecondCall().resolves() + @first = @ChannelManager.subscribe @rclient, "applied-ops", "1234567890abcdef" + # ignore error + @first.catch((()->)) + expect(@ChannelManager.getClientMapEntry(@rclient).get("applied-ops:1234567890abcdef")).to.equal @first + + @rclient.unsubscribe = sinon.stub().resolves() + @ChannelManager.unsubscribe @rclient, "applied-ops", "1234567890abcdef" + @second = @ChannelManager.subscribe @rclient, "applied-ops", "1234567890abcdef" + # should get replaced immediately + expect(@ChannelManager.getClientMapEntry(@rclient).get("applied-ops:1234567890abcdef")).to.equal @second + + # let the first subscribe error -> unsubscribe -> subscribe + setTimeout done + + it "should cleanup the second subscribePromise", -> + expect(@ChannelManager.getClientMapEntry(@rclient).has("applied-ops:1234567890abcdef")).to.equal false describe "when there is an existing subscription for another redis client but not this one", -> - beforeEach -> - @other_rclient.subscribe = sinon.stub() + beforeEach (done) -> + @other_rclient.subscribe = sinon.stub().resolves() @ChannelManager.subscribe @other_rclient, "applied-ops", "1234567890abcdef" - @rclient.subscribe = sinon.stub() # discard the original stub + @rclient.subscribe = sinon.stub().resolves() # discard the original stub @ChannelManager.subscribe @rclient, "applied-ops", "1234567890abcdef" + setTimeout done it "should subscribe to the redis channel on this redis client", -> @rclient.subscribe.calledWithExactly("applied-ops:1234567890abcdef").should.equal true @@ -46,30 +95,82 @@ describe 'ChannelManager', -> describe "unsubscribe", -> describe "when there is no existing subscription for this redis client", -> - beforeEach -> - @rclient.unsubscribe = sinon.stub() + beforeEach (done) -> + @rclient.unsubscribe = sinon.stub().resolves() @ChannelManager.unsubscribe @rclient, "applied-ops", "1234567890abcdef" + setTimeout done - it "should not unsubscribe from the redis channel", -> - @rclient.unsubscribe.called.should.equal false + it "should unsubscribe from the redis channel", -> + @rclient.unsubscribe.called.should.equal true describe "when there is an existing subscription for this another redis client but not this one", -> - beforeEach -> - @other_rclient.subscribe = sinon.stub() - @rclient.unsubscribe = sinon.stub() + beforeEach (done) -> + @other_rclient.subscribe = sinon.stub().resolves() + @rclient.unsubscribe = sinon.stub().resolves() @ChannelManager.subscribe @other_rclient, "applied-ops", "1234567890abcdef" @ChannelManager.unsubscribe @rclient, "applied-ops", "1234567890abcdef" + setTimeout done - it "should not unsubscribe from the redis channel on this client", -> - @rclient.unsubscribe.called.should.equal false + it "should still unsubscribe from the redis channel on this client", -> + @rclient.unsubscribe.called.should.equal true + + describe "when unsubscribe errors and completes", -> + beforeEach (done) -> + @rclient.subscribe = sinon.stub().resolves() + @ChannelManager.subscribe @rclient, "applied-ops", "1234567890abcdef" + @rclient.unsubscribe = sinon.stub().rejects(new Error("some redis error")) + @ChannelManager.unsubscribe @rclient, "applied-ops", "1234567890abcdef" + setTimeout done + return null + + it "should have cleaned up", -> + @ChannelManager.getClientMapEntry(@rclient).has("applied-ops:1234567890abcdef").should.equal false + + it "should not error out when subscribing again", (done) -> + p = @ChannelManager.subscribe @rclient, "applied-ops", "1234567890abcdef" + p.then () -> + done() + .catch done + return null + + describe "when unsubscribe errors and another client subscribes at the same time", -> + beforeEach (done) -> + @rclient.subscribe = sinon.stub().resolves() + @ChannelManager.subscribe @rclient, "applied-ops", "1234567890abcdef" + rejectSubscribe = undefined + @rclient.unsubscribe = () -> + return new Promise (resolve, reject) -> + rejectSubscribe = reject + @ChannelManager.unsubscribe @rclient, "applied-ops", "1234567890abcdef" + + setTimeout () => + # delay, actualUnsubscribe should not see the new subscribe request + @ChannelManager.subscribe(@rclient, "applied-ops", "1234567890abcdef") + .then () -> + setTimeout done + .catch done + setTimeout -> + # delay, rejectSubscribe is not defined immediately + rejectSubscribe(new Error("redis error")) + return null + + it "should have recorded the error", -> + expect(@metrics.inc.calledWithExactly("unsubscribe.failed.applied-ops")).to.equal(true) + + it "should have subscribed", -> + @rclient.subscribe.called.should.equal true + + it "should have discarded the finished Promise", -> + @ChannelManager.getClientMapEntry(@rclient).has("applied-ops:1234567890abcdef").should.equal false describe "when there is an existing subscription for this redis client", -> - beforeEach -> - @rclient.subscribe = sinon.stub() - @rclient.unsubscribe = sinon.stub() + beforeEach (done) -> + @rclient.subscribe = sinon.stub().resolves() + @rclient.unsubscribe = sinon.stub().resolves() @ChannelManager.subscribe @rclient, "applied-ops", "1234567890abcdef" @ChannelManager.unsubscribe @rclient, "applied-ops", "1234567890abcdef" + setTimeout done it "should unsubscribe from the redis channel", -> @rclient.unsubscribe.calledWithExactly("applied-ops:1234567890abcdef").should.equal true From 94d57f50c0095cfc01c5dd81cddf62eb63fa8ba9 Mon Sep 17 00:00:00 2001 From: Tim Alby Date: Thu, 28 May 2020 16:06:23 +0200 Subject: [PATCH 336/491] add fake lint and format targets Highly hacky! Lint and format steps are coming very soon thanks to the decaf, but in the meantime we need steps to pass CI. Updating the build scripts after the decaf will undo this change. --- services/real-time/Makefile | 3 +++ 1 file changed, 3 insertions(+) diff --git a/services/real-time/Makefile b/services/real-time/Makefile index 5dee366537..d8b68a699f 100644 --- a/services/real-time/Makefile +++ b/services/real-time/Makefile @@ -54,5 +54,8 @@ publish: docker push $(DOCKER_REPO)/$(PROJECT_NAME):$(BRANCH_NAME)-$(BUILD_NUMBER) +lint: + +format: .PHONY: clean test test_unit test_acceptance test_clean build publish From f973b377f0bd3ffb4c9b64f04bebd96b1f498a16 Mon Sep 17 00:00:00 2001 From: Brian Gough Date: Wed, 3 Jun 2020 09:12:21 +0100 Subject: [PATCH 337/491] update node to v10.21.0 --- services/real-time/.nvmrc | 2 +- services/real-time/Dockerfile | 2 +- services/real-time/buildscript.txt | 2 +- services/real-time/docker-compose.yml | 4 ++-- 4 files changed, 5 insertions(+), 5 deletions(-) diff --git a/services/real-time/.nvmrc b/services/real-time/.nvmrc index 5b7269c0a9..b61c07ffdd 100644 --- a/services/real-time/.nvmrc +++ b/services/real-time/.nvmrc @@ -1 +1 @@ -10.19.0 +10.21.0 diff --git a/services/real-time/Dockerfile b/services/real-time/Dockerfile index e538fb48d9..71e74fe251 100644 --- a/services/real-time/Dockerfile +++ b/services/real-time/Dockerfile @@ -3,7 +3,7 @@ # https://github.com/sharelatex/sharelatex-dev-environment # Version: 1.3.5 -FROM node:10.19.0 as base +FROM node:10.21.0 as base WORKDIR /app diff --git a/services/real-time/buildscript.txt b/services/real-time/buildscript.txt index 3e21ebbd0b..7e1b0350d4 100644 --- a/services/real-time/buildscript.txt +++ b/services/real-time/buildscript.txt @@ -2,7 +2,7 @@ real-time --public-repo=True --language=coffeescript --env-add= ---node-version=10.19.0 +--node-version=10.21.0 --acceptance-creds=None --dependencies=redis --docker-repos=gcr.io/overleaf-ops diff --git a/services/real-time/docker-compose.yml b/services/real-time/docker-compose.yml index 7a856c1ec4..b174363e67 100644 --- a/services/real-time/docker-compose.yml +++ b/services/real-time/docker-compose.yml @@ -7,7 +7,7 @@ version: "2.3" services: test_unit: - image: node:10.19.0 + image: node:10.21.0 volumes: - .:/app working_dir: /app @@ -18,7 +18,7 @@ services: user: node test_acceptance: - image: node:10.19.0 + image: node:10.21.0 volumes: - .:/app working_dir: /app From c6d08647c73c777e1f442224d3c4291f1842ea11 Mon Sep 17 00:00:00 2001 From: Jakob Ackermann Date: Thu, 4 Jun 2020 15:52:13 +0100 Subject: [PATCH 338/491] [misc] socket.io: use a secondary publicId for public facing usages --- .../coffee/DocumentUpdaterController.coffee | 2 +- services/real-time/app/coffee/Router.coffee | 4 ++- .../app/coffee/WebsocketController.coffee | 12 +++---- .../app/coffee/WebsocketLoadBalancer.coffee | 2 +- services/real-time/package.json | 1 + .../acceptance/coffee/ApplyUpdateTests.coffee | 6 ++-- .../coffee/ClientTrackingTests.coffee | 6 ++-- .../acceptance/coffee/JoinProjectTests.coffee | 2 +- .../coffee/LeaveProjectTests.coffee | 4 +-- .../coffee/ReceiveUpdateTests.coffee | 4 +-- .../coffee/helpers/RealTimeClient.coffee | 2 ++ .../DocumentUpdaterControllerTests.coffee | 2 +- .../coffee/WebsocketControllerTests.coffee | 33 ++++++++++--------- .../unit/coffee/helpers/MockClient.coffee | 1 + 14 files changed, 44 insertions(+), 37 deletions(-) diff --git a/services/real-time/app/coffee/DocumentUpdaterController.coffee b/services/real-time/app/coffee/DocumentUpdaterController.coffee index 2611d484ad..24b6a7c525 100644 --- a/services/real-time/app/coffee/DocumentUpdaterController.coffee +++ b/services/real-time/app/coffee/DocumentUpdaterController.coffee @@ -69,7 +69,7 @@ module.exports = DocumentUpdaterController = # send messages only to unique clients (due to duplicate entries in io.sockets.clients) for client in clientList when not seen[client.id] seen[client.id] = true - if client.id == update.meta.source + if client.publicId == update.meta.source logger.log doc_id: doc_id, version: update.v, source: update.meta?.source, "distributing update to sender" client.emit "otUpdateApplied", v: update.v, doc: update.doc else if !update.dup # Duplicate ops should just be sent back to sending client for acknowledgement diff --git a/services/real-time/app/coffee/Router.coffee b/services/real-time/app/coffee/Router.coffee index fcdb0c93dc..0266c71eec 100644 --- a/services/real-time/app/coffee/Router.coffee +++ b/services/real-time/app/coffee/Router.coffee @@ -6,6 +6,7 @@ HttpController = require "./HttpController" HttpApiController = require "./HttpApiController" Utils = require "./Utils" bodyParser = require "body-parser" +base64id = require("base64id") basicAuth = require('basic-auth-connect') httpAuth = basicAuth (user, pass)-> @@ -67,7 +68,8 @@ module.exports = Router = return # send positive confirmation that the client has a valid connection - client.emit("connectionAccepted") + client.publicId = base64id.generateId() + client.emit("connectionAccepted", null, client.publicId) metrics.inc('socket-io.connection') metrics.gauge('socket-io.clients', io.sockets.clients()?.length) diff --git a/services/real-time/app/coffee/WebsocketController.coffee b/services/real-time/app/coffee/WebsocketController.coffee index 17d27b15b6..4cc3a26daa 100644 --- a/services/real-time/app/coffee/WebsocketController.coffee +++ b/services/real-time/app/coffee/WebsocketController.coffee @@ -45,7 +45,7 @@ module.exports = WebsocketController = callback null, project, privilegeLevel, WebsocketController.PROTOCOL_VERSION # No need to block for setting the user as connected in the cursor tracking - ConnectedUsersManager.updateUserPosition project_id, client.id, user, null, () -> + ConnectedUsersManager.updateUserPosition project_id, client.publicId, user, null, () -> # We want to flush a project if there are no more (local) connected clients # but we need to wait for the triggering client to disconnect. How long we wait @@ -58,10 +58,10 @@ module.exports = WebsocketController = metrics.inc "editor.leave-project" logger.log {project_id, user_id, client_id: client.id}, "client leaving project" - WebsocketLoadBalancer.emitToRoom project_id, "clientTracking.clientDisconnected", client.id + WebsocketLoadBalancer.emitToRoom project_id, "clientTracking.clientDisconnected", client.publicId # We can do this in the background - ConnectedUsersManager.markUserAsDisconnected project_id, client.id, (err) -> + ConnectedUsersManager.markUserAsDisconnected project_id, client.publicId, (err) -> if err? logger.error {err, project_id, user_id, client_id: client.id}, "error marking client as disconnected" @@ -143,7 +143,7 @@ module.exports = WebsocketController = if error? logger.warn {err: error, client_id: client.id, project_id, user_id}, "silently ignoring unauthorized updateClientPosition. Client likely hasn't called joinProject yet." return callback() - cursorData.id = client.id + cursorData.id = client.publicId cursorData.user_id = user_id if user_id? cursorData.email = email if email? # Don't store anonymous users in redis to avoid influx @@ -159,7 +159,7 @@ module.exports = WebsocketController = last_name else "" - ConnectedUsersManager.updateUserPosition(project_id, client.id, { + ConnectedUsersManager.updateUserPosition(project_id, client.publicId, { first_name: first_name, last_name: last_name, email: email, @@ -205,7 +205,7 @@ module.exports = WebsocketController = , 100 return callback(error) update.meta ||= {} - update.meta.source = client.id + update.meta.source = client.publicId update.meta.user_id = user_id metrics.inc "editor.doc-update", 0.3 diff --git a/services/real-time/app/coffee/WebsocketLoadBalancer.coffee b/services/real-time/app/coffee/WebsocketLoadBalancer.coffee index 12b7ef812b..9508ac8714 100644 --- a/services/real-time/app/coffee/WebsocketLoadBalancer.coffee +++ b/services/real-time/app/coffee/WebsocketLoadBalancer.coffee @@ -72,7 +72,7 @@ module.exports = WebsocketLoadBalancer = clientList = io.sockets.clients(message.room_id) logger.log {channel:channel, message: message.message, room_id: message.room_id, message_id: message._id, socketIoClients: (client.id for client in clientList)}, "refreshing client list" for client in clientList - ConnectedUsersManager.refreshClient(message.room_id, client.id) + ConnectedUsersManager.refreshClient(message.room_id, client.publicId) else if message.room_id? if message._id? && Settings.checkEventOrder status = EventLogger.checkEventOrder("editor-events", message._id, message) diff --git a/services/real-time/package.json b/services/real-time/package.json index 1e20ac28fc..356a78e712 100644 --- a/services/real-time/package.json +++ b/services/real-time/package.json @@ -22,6 +22,7 @@ }, "dependencies": { "async": "^0.9.0", + "base64id": "0.1.0", "basic-auth-connect": "^1.0.0", "body-parser": "^1.12.0", "connect-redis": "^2.1.0", diff --git a/services/real-time/test/acceptance/coffee/ApplyUpdateTests.coffee b/services/real-time/test/acceptance/coffee/ApplyUpdateTests.coffee index 7820a090d8..e7babeb81c 100644 --- a/services/real-time/test/acceptance/coffee/ApplyUpdateTests.coffee +++ b/services/real-time/test/acceptance/coffee/ApplyUpdateTests.coffee @@ -55,7 +55,7 @@ describe "applyOtUpdate", -> update = JSON.parse(update) update.op.should.deep.equal @update.op update.meta.should.deep.equal { - source: @client.socket.sessionid + source: @client.publicId user_id: @user_id } done() @@ -208,7 +208,7 @@ describe "applyOtUpdate", -> update = JSON.parse(update) update.op.should.deep.equal @comment_update.op update.meta.should.deep.equal { - source: @client.socket.sessionid + source: @client.publicId user_id: @user_id } done() @@ -219,4 +219,4 @@ describe "applyOtUpdate", -> (cb) => rclient.del "pending-updates-list", cb (cb) => rclient.del "DocsWithPendingUpdates", "#{@project_id}:#{@doc_id}", cb (cb) => rclient.del redisSettings.documentupdater.key_schema.pendingUpdates({@doc_id}), cb - ], done \ No newline at end of file + ], done diff --git a/services/real-time/test/acceptance/coffee/ClientTrackingTests.coffee b/services/real-time/test/acceptance/coffee/ClientTrackingTests.coffee index d13663ed70..e76eca7d8e 100644 --- a/services/real-time/test/acceptance/coffee/ClientTrackingTests.coffee +++ b/services/real-time/test/acceptance/coffee/ClientTrackingTests.coffee @@ -63,7 +63,7 @@ describe "clientTracking", -> row: @row column: @column doc_id: @doc_id - id: @clientA.socket.sessionid + id: @clientA.publicId user_id: @user_id name: "Joe Bloggs" } @@ -72,7 +72,7 @@ describe "clientTracking", -> it "should record the update in getConnectedUsers", (done) -> @clientB.emit "clientTracking.getConnectedUsers", (error, users) => for user in users - if user.client_id == @clientA.socket.sessionid + if user.client_id == @clientA.publicId expect(user.cursorData).to.deep.equal({ row: @row column: @column @@ -139,7 +139,7 @@ describe "clientTracking", -> row: @row column: @column doc_id: @doc_id - id: @anonymous.socket.sessionid + id: @anonymous.publicId user_id: "anonymous-user" name: "" } diff --git a/services/real-time/test/acceptance/coffee/JoinProjectTests.coffee b/services/real-time/test/acceptance/coffee/JoinProjectTests.coffee index b3707e0981..11082cdb6c 100644 --- a/services/real-time/test/acceptance/coffee/JoinProjectTests.coffee +++ b/services/real-time/test/acceptance/coffee/JoinProjectTests.coffee @@ -55,7 +55,7 @@ describe "joinProject", -> @client.emit "clientTracking.getConnectedUsers", (error, users) => connected = false for user in users - if user.client_id == @client.socket.sessionid and user.user_id == @user_id + if user.client_id == @client.publicId and user.user_id == @user_id connected = true break expect(connected).to.equal true diff --git a/services/real-time/test/acceptance/coffee/LeaveProjectTests.coffee b/services/real-time/test/acceptance/coffee/LeaveProjectTests.coffee index 5925472245..91ec1a1159 100644 --- a/services/real-time/test/acceptance/coffee/LeaveProjectTests.coffee +++ b/services/real-time/test/acceptance/coffee/LeaveProjectTests.coffee @@ -64,12 +64,12 @@ describe "leaveProject", -> ], done it "should emit a disconnect message to the room", -> - @clientBDisconnectMessages.should.deep.equal [@clientA.socket.sessionid] + @clientBDisconnectMessages.should.deep.equal [@clientA.publicId] it "should no longer list the client in connected users", (done) -> @clientB.emit "clientTracking.getConnectedUsers", (error, users) => for user in users - if user.client_id == @clientA.socket.sessionid + if user.client_id == @clientA.publicId throw "Expected clientA to not be listed in connected users" return done() diff --git a/services/real-time/test/acceptance/coffee/ReceiveUpdateTests.coffee b/services/real-time/test/acceptance/coffee/ReceiveUpdateTests.coffee index ec2c26ca4e..fb504dd9aa 100644 --- a/services/real-time/test/acceptance/coffee/ReceiveUpdateTests.coffee +++ b/services/real-time/test/acceptance/coffee/ReceiveUpdateTests.coffee @@ -65,7 +65,7 @@ describe "receiveUpdate", -> doc_id: @doc_id op: meta: - source: @clientA.socket.sessionid + source: @clientA.publicId v: @version doc: @doc_id op: [{i: "foo", p: 50}] @@ -97,4 +97,4 @@ describe "receiveUpdate", -> it "should disconnect the clients", -> @clientA.socket.connected.should.equal false - @clientB.socket.connected.should.equal false \ No newline at end of file + @clientB.socket.connected.should.equal false diff --git a/services/real-time/test/acceptance/coffee/helpers/RealTimeClient.coffee b/services/real-time/test/acceptance/coffee/helpers/RealTimeClient.coffee index 21da045e83..e3254a79a5 100644 --- a/services/real-time/test/acceptance/coffee/helpers/RealTimeClient.coffee +++ b/services/real-time/test/acceptance/coffee/helpers/RealTimeClient.coffee @@ -37,6 +37,8 @@ module.exports = Client = connect: (cookie) -> client = io.connect("http://localhost:3026", 'force new connection': true) + client.on 'connectionAccepted', (_, publicId) -> + client.publicId = publicId return client getConnectedClients: (callback = (error, clients) ->) -> diff --git a/services/real-time/test/unit/coffee/DocumentUpdaterControllerTests.coffee b/services/real-time/test/unit/coffee/DocumentUpdaterControllerTests.coffee index b5574c8d16..b2e52c7d56 100644 --- a/services/real-time/test/unit/coffee/DocumentUpdaterControllerTests.coffee +++ b/services/real-time/test/unit/coffee/DocumentUpdaterControllerTests.coffee @@ -93,7 +93,7 @@ describe "DocumentUpdaterController", -> @otherClients = [new MockClient(), new MockClient()] @update = op: [ t: "foo", p: 12 ] - meta: source: @sourceClient.id + meta: source: @sourceClient.publicId v: @version = 42 doc: @doc_id @io.sockets = diff --git a/services/real-time/test/unit/coffee/WebsocketControllerTests.coffee b/services/real-time/test/unit/coffee/WebsocketControllerTests.coffee index 498b425281..6416ded845 100644 --- a/services/real-time/test/unit/coffee/WebsocketControllerTests.coffee +++ b/services/real-time/test/unit/coffee/WebsocketControllerTests.coffee @@ -21,6 +21,7 @@ describe 'WebsocketController', -> @callback = sinon.stub() @client = id: @client_id = "mock-client-id-123" + publicId: "other-id-#{Math.random()}" params: {} set: sinon.stub() get: (param, cb) -> cb null, @params[param] @@ -106,7 +107,7 @@ describe 'WebsocketController', -> it "should mark the user as connected in ConnectedUsersManager", -> @ConnectedUsersManager.updateUserPosition - .calledWith(@project_id, @client.id, @user, null) + .calledWith(@project_id, @client.publicId, @user, null) .should.equal true it "should increment the join-project metric", -> @@ -185,12 +186,12 @@ describe 'WebsocketController', -> it "should end clientTracking.clientDisconnected to the project room", -> @WebsocketLoadBalancer.emitToRoom - .calledWith(@project_id, "clientTracking.clientDisconnected", @client.id) + .calledWith(@project_id, "clientTracking.clientDisconnected", @client.publicId) .should.equal true it "should mark the user as disconnected", -> @ConnectedUsersManager.markUserAsDisconnected - .calledWith(@project_id, @client.id) + .calledWith(@project_id, @client.publicId) .should.equal true it "should flush the project in the document updater", -> @@ -223,12 +224,12 @@ describe 'WebsocketController', -> it "should not end clientTracking.clientDisconnected to the project room", -> @WebsocketLoadBalancer.emitToRoom - .calledWith(@project_id, "clientTracking.clientDisconnected", @client.id) + .calledWith(@project_id, "clientTracking.clientDisconnected", @client.publicId) .should.equal false it "should not mark the user as disconnected", -> @ConnectedUsersManager.markUserAsDisconnected - .calledWith(@project_id, @client.id) + .calledWith(@project_id, @client.publicId) .should.equal false it "should not flush the project in the document updater", -> @@ -247,12 +248,12 @@ describe 'WebsocketController', -> it "should not end clientTracking.clientDisconnected to the project room", -> @WebsocketLoadBalancer.emitToRoom - .calledWith(@project_id, "clientTracking.clientDisconnected", @client.id) + .calledWith(@project_id, "clientTracking.clientDisconnected", @client.publicId) .should.equal false it "should not mark the user as disconnected", -> @ConnectedUsersManager.markUserAsDisconnected - .calledWith(@project_id, @client.id) + .calledWith(@project_id, @client.publicId) .should.equal false it "should not flush the project in the document updater", -> @@ -488,7 +489,7 @@ describe 'WebsocketController', -> @populatedCursorData = doc_id: @doc_id, - id: @client.id + id: @client.publicId name: "#{@first_name} #{@last_name}" row: @row column: @column @@ -499,7 +500,7 @@ describe 'WebsocketController', -> @WebsocketLoadBalancer.emitToRoom.calledWith(@project_id, "clientTracking.clientUpdated", @populatedCursorData).should.equal true it "should send the cursor data to the connected user manager", (done)-> - @ConnectedUsersManager.updateUserPosition.calledWith(@project_id, @client.id, { + @ConnectedUsersManager.updateUserPosition.calledWith(@project_id, @client.publicId, { _id: @user_id, email: @email, first_name: @first_name, @@ -528,7 +529,7 @@ describe 'WebsocketController', -> @populatedCursorData = doc_id: @doc_id, - id: @client.id + id: @client.publicId name: "#{@first_name}" row: @row column: @column @@ -539,7 +540,7 @@ describe 'WebsocketController', -> @WebsocketLoadBalancer.emitToRoom.calledWith(@project_id, "clientTracking.clientUpdated", @populatedCursorData).should.equal true it "should send the cursor data to the connected user manager", (done)-> - @ConnectedUsersManager.updateUserPosition.calledWith(@project_id, @client.id, { + @ConnectedUsersManager.updateUserPosition.calledWith(@project_id, @client.publicId, { _id: @user_id, email: @email, first_name: @first_name, @@ -568,7 +569,7 @@ describe 'WebsocketController', -> @populatedCursorData = doc_id: @doc_id, - id: @client.id + id: @client.publicId name: "#{@last_name}" row: @row column: @column @@ -579,7 +580,7 @@ describe 'WebsocketController', -> @WebsocketLoadBalancer.emitToRoom.calledWith(@project_id, "clientTracking.clientUpdated", @populatedCursorData).should.equal true it "should send the cursor data to the connected user manager", (done)-> - @ConnectedUsersManager.updateUserPosition.calledWith(@project_id, @client.id, { + @ConnectedUsersManager.updateUserPosition.calledWith(@project_id, @client.publicId, { _id: @user_id, email: @email, first_name: undefined, @@ -609,7 +610,7 @@ describe 'WebsocketController', -> @WebsocketLoadBalancer.emitToRoom .calledWith(@project_id, "clientTracking.clientUpdated", { doc_id: @doc_id, - id: @client.id, + id: @client.publicId, user_id: @user_id, name: "", row: @row, @@ -631,7 +632,7 @@ describe 'WebsocketController', -> @WebsocketLoadBalancer.emitToRoom .calledWith(@project_id, "clientTracking.clientUpdated", { doc_id: @doc_id, - id: @client.id + id: @client.publicId name: "" row: @row column: @column @@ -655,7 +656,7 @@ describe 'WebsocketController', -> @WebsocketController.applyOtUpdate @client, @doc_id, @update, @callback it "should set the source of the update to the client id", -> - @update.meta.source.should.equal @client.id + @update.meta.source.should.equal @client.publicId it "should set the user_id of the update to the user id", -> @update.meta.user_id.should.equal @user_id diff --git a/services/real-time/test/unit/coffee/helpers/MockClient.coffee b/services/real-time/test/unit/coffee/helpers/MockClient.coffee index 82c3c02b19..4dcddeb01d 100644 --- a/services/real-time/test/unit/coffee/helpers/MockClient.coffee +++ b/services/real-time/test/unit/coffee/helpers/MockClient.coffee @@ -9,6 +9,7 @@ module.exports = class MockClient @emit = sinon.stub() @disconnect = sinon.stub() @id = idCounter++ + @publicId = idCounter++ set : (key, value, callback) -> @attributes[key] = value callback() if callback? From 0840700ffd0954af98b11d391bd0d4cf2a58da75 Mon Sep 17 00:00:00 2001 From: Jakob Ackermann Date: Fri, 22 May 2020 10:17:50 +0200 Subject: [PATCH 339/491] [Router] validate the callback argument --- services/real-time/app/coffee/Router.coffee | 32 +++++++++++++++++++-- 1 file changed, 29 insertions(+), 3 deletions(-) diff --git a/services/real-time/app/coffee/Router.coffee b/services/real-time/app/coffee/Router.coffee index fcdb0c93dc..7714ab0ad0 100644 --- a/services/real-time/app/coffee/Router.coffee +++ b/services/real-time/app/coffee/Router.coffee @@ -24,6 +24,10 @@ module.exports = Router = if error.name == "CodedError" logger.warn attrs, error.message, code: error.code return callback {message: error.message, code: error.code} + if error.message == 'unexpected arguments' + logger.log attrs, 'unexpected arguments' + metrics.inc 'unexpected-arguments', 1, { status: method } + return callback { message: error.message } if error.message in ["not authorized", "doc updater could not load requested ops", "no project_id found on client"] logger.warn attrs, error.message return callback {message: error.message} @@ -32,6 +36,14 @@ module.exports = Router = # Don't return raw error to prevent leaking server side info return callback {message: "Something went wrong in real-time service"} + _handleInvalidArguments: (client, method, args) -> + error = new Error("unexpected arguments") + callback = args[args.length - 1] + if typeof callback != 'function' + callback = (() ->) + attrs = {arguments: args} + Router._handleError(callback, error, client, method, attrs) + configure: (app, io, session) -> app.set("io", io) app.get "/clients", HttpController.getConnectedClients @@ -82,6 +94,9 @@ module.exports = Router = user = {_id: "anonymous-user"} client.on "joinProject", (data = {}, callback) -> + if typeof callback != 'function' + return Router._handleInvalidArguments(client, 'joinProject', arguments) + if data.anonymousAccessToken user.anonymousAccessToken = data.anonymousAccessToken WebsocketController.joinProject client, user, data.project_id, (err, args...) -> @@ -114,11 +129,10 @@ module.exports = Router = callback = options options = fromVersion fromVersion = -1 - else if typeof fromVersion == "number" and typeof options == "object" + else if typeof fromVersion == "number" and typeof options == "object" and typeof callback == 'function' # Called with 4 args, things are as expected else - logger.error { arguments: arguments }, "unexpected arguments" - return callback?(new Error("unexpected arguments")) + return Router._handleInvalidArguments(client, 'joinDoc', arguments) WebsocketController.joinDoc client, doc_id, fromVersion, options, (err, args...) -> if err? @@ -127,6 +141,9 @@ module.exports = Router = callback(null, args...) client.on "leaveDoc", (doc_id, callback) -> + if typeof callback != 'function' + return Router._handleInvalidArguments(client, 'leaveDoc', arguments) + WebsocketController.leaveDoc client, doc_id, (err, args...) -> if err? Router._handleError callback, err, client, "leaveDoc" @@ -134,6 +151,9 @@ module.exports = Router = callback(null, args...) client.on "clientTracking.getConnectedUsers", (callback = (error, users) ->) -> + if typeof callback != 'function' + return Router._handleInvalidArguments(client, 'clientTracking.getConnectedUsers', arguments) + WebsocketController.getConnectedUsers client, (err, users) -> if err? Router._handleError callback, err, client, "clientTracking.getConnectedUsers" @@ -141,6 +161,9 @@ module.exports = Router = callback(null, users) client.on "clientTracking.updatePosition", (cursorData, callback = (error) ->) -> + if typeof callback != 'function' + return Router._handleInvalidArguments(client, 'clientTracking.updatePosition', arguments) + WebsocketController.updateClientPosition client, cursorData, (err) -> if err? Router._handleError callback, err, client, "clientTracking.updatePosition" @@ -148,6 +171,9 @@ module.exports = Router = callback() client.on "applyOtUpdate", (doc_id, update, callback = (error) ->) -> + if typeof callback != 'function' + return Router._handleInvalidArguments(client, 'applyOtUpdate', arguments) + WebsocketController.applyOtUpdate client, doc_id, update, (err) -> if err? Router._handleError callback, err, client, "applyOtUpdate", {doc_id, update} From 7fa906101598e2d53a608bc60adeda691a7e82a4 Mon Sep 17 00:00:00 2001 From: Jakob Ackermann Date: Tue, 28 Apr 2020 17:03:38 +0100 Subject: [PATCH 340/491] [misc] stop processing requests as we detect a client disconnect v2 expose `client.connected`; v0 exposes `client.disconnected` (cherry-picked from commit a9d70484343ca9be367d45bf7bb949e4be449647) --- .../app/coffee/WebsocketController.coffee | 33 ++++ .../acceptance/coffee/EarlyDisconnect.coffee | 160 ++++++++++++++++++ .../coffee/WebsocketControllerTests.coffee | 127 +++++++++++++- 3 files changed, 315 insertions(+), 5 deletions(-) create mode 100644 services/real-time/test/acceptance/coffee/EarlyDisconnect.coffee diff --git a/services/real-time/app/coffee/WebsocketController.coffee b/services/real-time/app/coffee/WebsocketController.coffee index 17d27b15b6..bd83b0c19a 100644 --- a/services/real-time/app/coffee/WebsocketController.coffee +++ b/services/real-time/app/coffee/WebsocketController.coffee @@ -16,11 +16,18 @@ module.exports = WebsocketController = PROTOCOL_VERSION: 2 joinProject: (client, user, project_id, callback = (error, project, privilegeLevel, protocolVersion) ->) -> + if client.disconnected + metrics.inc('disconnected_join_project') + return callback() + user_id = user?._id logger.log {user_id, project_id, client_id: client.id}, "user joining project" metrics.inc "editor.join-project" WebApiManager.joinProject project_id, user, (error, project, privilegeLevel, isRestrictedUser) -> return callback(error) if error? + if client.disconnected + metrics.inc('disconnected_join_project') + return callback() if !privilegeLevel or privilegeLevel == "" err = new Error("not authorized") @@ -77,6 +84,10 @@ module.exports = WebsocketController = , WebsocketController.FLUSH_IF_EMPTY_DELAY joinDoc: (client, doc_id, fromVersion = -1, options, callback = (error, doclines, version, ops, ranges) ->) -> + if client.disconnected + metrics.inc('disconnected_join_doc') + return callback() + metrics.inc "editor.join-doc" Utils.getClientAttributes client, ["project_id", "user_id", "is_restricted_user"], (error, {project_id, user_id, is_restricted_user}) -> return callback(error) if error? @@ -89,8 +100,17 @@ module.exports = WebsocketController = # doc to the client, so that no events are missed. RoomManager.joinDoc client, doc_id, (error) -> return callback(error) if error? + if client.disconnected + metrics.inc('disconnected_join_doc') + # the client will not read the response anyways + return callback() + DocumentUpdaterManager.getDocument project_id, doc_id, fromVersion, (error, lines, version, ranges, ops) -> return callback(error) if error? + if client.disconnected + metrics.inc('disconnected_join_doc') + # the client will not read the response anyways + return callback() if is_restricted_user and ranges?.comments? ranges.comments = [] @@ -122,6 +142,7 @@ module.exports = WebsocketController = callback null, escapedLines, version, ops, ranges leaveDoc: (client, doc_id, callback = (error) ->) -> + # client may have disconnected, but we have to cleanup internal state. metrics.inc "editor.leave-doc" Utils.getClientAttributes client, ["project_id", "user_id"], (error, {project_id, user_id}) -> logger.log {user_id, project_id, doc_id, client_id: client.id}, "client leaving doc" @@ -132,6 +153,10 @@ module.exports = WebsocketController = ## AuthorizationManager.removeAccessToDoc client, doc_id callback() updateClientPosition: (client, cursorData, callback = (error) ->) -> + if client.disconnected + # do not create a ghost entry in redis + return callback() + metrics.inc "editor.update-client-position", 0.1 Utils.getClientAttributes client, [ "project_id", "first_name", "last_name", "email", "user_id" @@ -173,6 +198,10 @@ module.exports = WebsocketController = CLIENT_REFRESH_DELAY: 1000 getConnectedUsers: (client, callback = (error, users) ->) -> + if client.disconnected + # they are not interested anymore, skip the redis lookups + return callback() + metrics.inc "editor.get-connected-users" Utils.getClientAttributes client, ["project_id", "user_id", "is_restricted_user"], (error, clientAttributes) -> return callback(error) if error? @@ -192,6 +221,7 @@ module.exports = WebsocketController = , WebsocketController.CLIENT_REFRESH_DELAY applyOtUpdate: (client, doc_id, update, callback = (error) ->) -> + # client may have disconnected, but we can submit their update to doc-updater anyways. Utils.getClientAttributes client, ["user_id", "project_id"], (error, {user_id, project_id}) -> return callback(error) if error? return callback(new Error("no project_id found on client")) if !project_id? @@ -223,6 +253,9 @@ module.exports = WebsocketController = # trigger an out-of-sync error message = {project_id, doc_id, error: "update is too large"} setTimeout () -> + if client.disconnected + # skip the message broadcast, the client has moved on + return metrics.inc('disconnected_otUpdateError') client.emit "otUpdateError", message.error, message client.disconnect() , 100 diff --git a/services/real-time/test/acceptance/coffee/EarlyDisconnect.coffee b/services/real-time/test/acceptance/coffee/EarlyDisconnect.coffee new file mode 100644 index 0000000000..d90c36b430 --- /dev/null +++ b/services/real-time/test/acceptance/coffee/EarlyDisconnect.coffee @@ -0,0 +1,160 @@ +async = require "async" +{expect} = require("chai") + +RealTimeClient = require "./helpers/RealTimeClient" +MockDocUpdaterServer = require "./helpers/MockDocUpdaterServer" +MockWebServer = require "./helpers/MockWebServer" +FixturesManager = require "./helpers/FixturesManager" + +settings = require "settings-sharelatex" +redis = require "redis-sharelatex" +rclient = redis.createClient(settings.redis.pubsub) +rclientRT = redis.createClient(settings.redis.realtime) +KeysRT = settings.redis.realtime.key_schema + +describe "EarlyDisconnect", -> + before (done) -> + MockDocUpdaterServer.run done + + describe "when the client disconnects before joinProject completes", -> + before () -> + # slow down web-api requests to force the race condition + @actualWebAPIjoinProject = joinProject = MockWebServer.joinProject + MockWebServer.joinProject = (project_id, user_id, cb) -> + setTimeout () -> + joinProject(project_id, user_id, cb) + , 300 + + after () -> + MockWebServer.joinProject = @actualWebAPIjoinProject + + beforeEach (done) -> + async.series [ + (cb) => + FixturesManager.setUpProject { + privilegeLevel: "owner" + project: { + name: "Test Project" + } + }, (e, {@project_id, @user_id}) => cb() + + (cb) => + @clientA = RealTimeClient.connect() + @clientA.on "connectionAccepted", cb + + (cb) => + @clientA.emit "joinProject", project_id: @project_id, (() ->) + # disconnect before joinProject completes + @clientA.on "disconnect", () -> cb() + @clientA.disconnect() + + (cb) => + # wait for joinDoc and subscribe + setTimeout cb, 500 + ], done + + # we can force the race condition, there is no need to repeat too often + for attempt in Array.from(length: 5).map((_, i) -> i+1) + it "should not subscribe to the pub/sub channel anymore (race #{attempt})", (done) -> + rclient.pubsub 'CHANNELS', (err, resp) => + return done(err) if err + expect(resp).to.not.include "editor-events:#{@project_id}" + done() + return null + + describe "when the client disconnects before joinDoc completes", -> + beforeEach (done) -> + async.series [ + (cb) => + FixturesManager.setUpProject { + privilegeLevel: "owner" + project: { + name: "Test Project" + } + }, (e, {@project_id, @user_id}) => cb() + + (cb) => + @clientA = RealTimeClient.connect() + @clientA.on "connectionAccepted", cb + + (cb) => + @clientA.emit "joinProject", project_id: @project_id, (error, @project, @privilegeLevel, @protocolVersion) => + cb(error) + + (cb) => + FixturesManager.setUpDoc @project_id, {@lines, @version, @ops}, (e, {@doc_id}) => + cb(e) + + (cb) => + @clientA.emit "joinDoc", @doc_id, (() ->) + # disconnect before joinDoc completes + @clientA.on "disconnect", () -> cb() + @clientA.disconnect() + + (cb) => + # wait for subscribe and unsubscribe + setTimeout cb, 100 + ], done + + # we can not force the race condition, so we have to try many times + for attempt in Array.from(length: 20).map((_, i) -> i+1) + it "should not subscribe to the pub/sub channels anymore (race #{attempt})", (done) -> + rclient.pubsub 'CHANNELS', (err, resp) => + return done(err) if err + expect(resp).to.not.include "editor-events:#{@project_id}" + + rclient.pubsub 'CHANNELS', (err, resp) => + return done(err) if err + expect(resp).to.not.include "applied-ops:#{@doc_id}" + done() + return null + + describe "when the client disconnects before clientTracking.updatePosition starts", -> + beforeEach (done) -> + async.series [ + (cb) => + FixturesManager.setUpProject { + privilegeLevel: "owner" + project: { + name: "Test Project" + } + }, (e, {@project_id, @user_id}) => cb() + + (cb) => + @clientA = RealTimeClient.connect() + @clientA.on "connectionAccepted", cb + + (cb) => + @clientA.emit "joinProject", project_id: @project_id, (error, @project, @privilegeLevel, @protocolVersion) => + cb(error) + + (cb) => + FixturesManager.setUpDoc @project_id, {@lines, @version, @ops}, (e, {@doc_id}) => + cb(e) + + (cb) => + @clientA.emit "joinDoc", @doc_id, cb + + (cb) => + @clientA.emit "clientTracking.updatePosition", { + row: 42 + column: 36 + doc_id: @doc_id + }, (() ->) + # disconnect before updateClientPosition completes + @clientA.on "disconnect", () -> cb() + @clientA.disconnect() + + (cb) => + # wait for updateClientPosition + setTimeout cb, 100 + ], done + + # we can not force the race condition, so we have to try many times + for attempt in Array.from(length: 20).map((_, i) -> i+1) + it "should not show the client as connected (race #{attempt})", (done) -> + rclientRT.smembers KeysRT.clientsInProject({project_id: @project_id}), (err, results) -> + return done(err) if err + expect(results).to.deep.equal([]) + done() + return null diff --git a/services/real-time/test/unit/coffee/WebsocketControllerTests.coffee b/services/real-time/test/unit/coffee/WebsocketControllerTests.coffee index 498b425281..3819b04d7b 100644 --- a/services/real-time/test/unit/coffee/WebsocketControllerTests.coffee +++ b/services/real-time/test/unit/coffee/WebsocketControllerTests.coffee @@ -20,6 +20,7 @@ describe 'WebsocketController', -> } @callback = sinon.stub() @client = + disconnected: false id: @client_id = "mock-client-id-123" params: {} set: sinon.stub() @@ -147,6 +148,35 @@ describe 'WebsocketController', -> .should.equal true @callback.args[0][0].message.should.equal "subscribe failed" + describe "when the client has disconnected", -> + beforeEach -> + @client.disconnected = true + @WebApiManager.joinProject = sinon.stub().callsArg(2) + @WebsocketController.joinProject @client, @user, @project_id, @callback + + it "should not call WebApiManager.joinProject", -> + expect(@WebApiManager.joinProject.called).to.equal(false) + + it "should call the callback with no details", -> + expect(@callback.args[0]).to.deep.equal [] + + it "should increment the disconnected_join_project metric", -> + expect(@metrics.inc.calledWith("disconnected_join_project")).to.equal(true) + + describe "when the client disconnects while WebApiManager.joinProject is running", -> + beforeEach -> + @WebApiManager.joinProject = (project, user, cb) => + @client.disconnected = true + cb(null, @project, @privilegeLevel, @isRestrictedUser) + + @WebsocketController.joinProject @client, @user, @project_id, @callback + + it "should call the callback with no details", -> + expect(@callback.args[0]).to.deep.equal [] + + it "should increment the disconnected_join_project metric", -> + expect(@metrics.inc.calledWith("disconnected_join_project")).to.equal(true) + describe "leaveProject", -> beforeEach -> @DocumentUpdaterManager.flushProjectToMongoAndDelete = sinon.stub().callsArg(1) @@ -384,6 +414,51 @@ describe 'WebsocketController', -> ranges = @callback.args[0][4] expect(ranges.comments).to.deep.equal [] + describe "when the client has disconnected", -> + beforeEach -> + @client.disconnected = true + @WebsocketController.joinDoc @client, @doc_id, -1, @options, @callback + + it "should call the callback with no details", -> + expect(@callback.args[0]).to.deep.equal([]) + + it "should increment the disconnected_join_doc metric", -> + expect(@metrics.inc.calledWith("disconnected_join_doc")).to.equal(true) + + it "should not get the document", -> + expect(@DocumentUpdaterManager.getDocument.called).to.equal(false) + + describe "when the client disconnects while RoomManager.joinDoc is running", -> + beforeEach -> + @RoomManager.joinDoc = (client, doc_id, cb) => + @client.disconnected = true + cb() + + @WebsocketController.joinDoc @client, @doc_id, -1, @options, @callback + + it "should call the callback with no details", -> + expect(@callback.args[0]).to.deep.equal([]) + + it "should increment the disconnected_join_doc metric", -> + expect(@metrics.inc.calledWith("disconnected_join_doc")).to.equal(true) + + it "should not get the document", -> + expect(@DocumentUpdaterManager.getDocument.called).to.equal(false) + + describe "when the client disconnects while DocumentUpdaterManager.getDocument is running", -> + beforeEach -> + @DocumentUpdaterManager.getDocument = (project_id, doc_id, fromVersion, callback) => + @client.disconnected = true + callback(null, @doc_lines, @version, @ranges, @ops) + + @WebsocketController.joinDoc @client, @doc_id, -1, @options, @callback + + it "should call the callback with no details", -> + expect(@callback.args[0]).to.deep.equal [] + + it "should increment the disconnected_join_doc metric", -> + expect(@metrics.inc.calledWith("disconnected_join_doc")).to.equal(true) + describe "leaveDoc", -> beforeEach -> @doc_id = "doc-id-123" @@ -463,6 +538,18 @@ describe 'WebsocketController', -> .called .should.equal false + describe "when the client has disconnected", -> + beforeEach -> + @client.disconnected = true + @AuthorizationManager.assertClientCanViewProject = sinon.stub() + @WebsocketController.getConnectedUsers @client, @callback + + it "should call the callback with no details", -> + expect(@callback.args[0]).to.deep.equal([]) + + it "should not check permissions", -> + expect(@AuthorizationManager.assertClientCanViewProject.called).to.equal(false) + describe "updateClientPosition", -> beforeEach -> @WebsocketLoadBalancer.emitToRoom = sinon.stub() @@ -642,6 +729,18 @@ describe 'WebsocketController', -> @ConnectedUsersManager.updateUserPosition.called.should.equal false done() + describe "when the client has disconnected", -> + beforeEach -> + @client.disconnected = true + @AuthorizationManager.assertClientCanViewProjectAndDoc = sinon.stub() + @WebsocketController.updateClientPosition @client, @update, @callback + + it "should call the callback with no details", -> + expect(@callback.args[0]).to.deep.equal([]) + + it "should not check permissions", -> + expect(@AuthorizationManager.assertClientCanViewProjectAndDoc.called).to.equal(false) + describe "applyOtUpdate", -> beforeEach -> @update = {op: {p: 12, t: "foo"}} @@ -715,7 +814,7 @@ describe 'WebsocketController', -> @WebsocketController.applyOtUpdate @client, @doc_id, @update, @callback setTimeout -> done() - , 201 + , 1 it "should call the callback with no error", -> @callback.called.should.equal true @@ -727,11 +826,29 @@ describe 'WebsocketController', -> @user_id, @project_id, @doc_id, updateSize: 7372835 }, 'update is too large'] - it "should send an otUpdateError the client", -> - @client.emit.calledWith('otUpdateError').should.equal true + describe "after 100ms", -> + beforeEach (done) -> + setTimeout done, 100 - it "should disconnect the client", -> - @client.disconnect.called.should.equal true + it "should send an otUpdateError the client", -> + @client.emit.calledWith('otUpdateError').should.equal true + + it "should disconnect the client", -> + @client.disconnect.called.should.equal true + + describe "when the client disconnects during the next 100ms", -> + beforeEach (done) -> + @client.disconnected = true + setTimeout done, 100 + + it "should not send an otUpdateError the client", -> + @client.emit.calledWith('otUpdateError').should.equal false + + it "should not disconnect the client", -> + @client.disconnect.called.should.equal false + + it "should increment the disconnected_otUpdateError metric", -> + expect(@metrics.inc.calledWith("disconnected_otUpdateError")).to.equal(true) describe "_assertClientCanApplyUpdate", -> beforeEach -> From ddcb9cf8c86102bbec057b82b683f85dc8ce94df Mon Sep 17 00:00:00 2001 From: Jakob Ackermann Date: Tue, 19 May 2020 17:41:20 +0100 Subject: [PATCH 341/491] [misc] downgrade a warning message from clients leaving non-joined rooms This can now happen all the time, as we skip the join for clients that disconnect before joinProject/joinDoc completed. (cherry-picked from commit f357931de74e088800f3cced3898cce4f251dad0) --- services/real-time/app/coffee/RoomManager.coffee | 4 +++- .../real-time/test/acceptance/coffee/LeaveDocTests.coffee | 8 +++++--- 2 files changed, 8 insertions(+), 4 deletions(-) diff --git a/services/real-time/app/coffee/RoomManager.coffee b/services/real-time/app/coffee/RoomManager.coffee index 27d69ca379..cad3fd0a12 100644 --- a/services/real-time/app/coffee/RoomManager.coffee +++ b/services/real-time/app/coffee/RoomManager.coffee @@ -71,8 +71,10 @@ module.exports = RoomManager = # Ignore any requests to leave when the client is not actually in the # room. This can happen if the client sends spurious leaveDoc requests # for old docs after a reconnection. + # This can now happen all the time, as we skip the join for clients that + # disconnect before joinProject/joinDoc completed. if !@_clientAlreadyInRoom(client, id) - logger.warn {client: client.id, entity, id}, "ignoring request from client to leave room it is not in" + logger.log {client: client.id, entity, id}, "ignoring request from client to leave room it is not in" return client.leave id afterCount = @_clientsInRoom(client, id) diff --git a/services/real-time/test/acceptance/coffee/LeaveDocTests.coffee b/services/real-time/test/acceptance/coffee/LeaveDocTests.coffee index e68b111c64..e35e9093d3 100644 --- a/services/real-time/test/acceptance/coffee/LeaveDocTests.coffee +++ b/services/real-time/test/acceptance/coffee/LeaveDocTests.coffee @@ -17,12 +17,14 @@ describe "leaveDoc", -> @ops = ["mock", "doc", "ops"] sinon.spy(logger, "error") sinon.spy(logger, "warn") + sinon.spy(logger, "log") @other_doc_id = FixturesManager.getRandomId() after -> logger.error.restore() # remove the spy logger.warn.restore() - + logger.log.restore() + describe "when joined to a doc", -> beforeEach (done) -> async.series [ @@ -80,5 +82,5 @@ describe "leaveDoc", -> throw error if error? done() - it "should trigger a warning only", -> - sinon.assert.calledWith(logger.warn, sinon.match.any, "ignoring request from client to leave room it is not in") \ No newline at end of file + it "should trigger a low level message only", -> + sinon.assert.calledWith(logger.log, sinon.match.any, "ignoring request from client to leave room it is not in") From 0b2cccf1e02ae8d2d08ef1745717609643261e7e Mon Sep 17 00:00:00 2001 From: Jakob Ackermann Date: Thu, 21 May 2020 10:08:23 +0100 Subject: [PATCH 342/491] [misc] apply review feedback: adjust metric names Co-Authored-By: Brian Gough (cherry-picked from commit 67674b83efb452ece05cdc39525ee3a5eeb8a4d7) --- .../app/coffee/WebsocketController.coffee | 12 +++++----- .../coffee/WebsocketControllerTests.coffee | 24 +++++++++---------- 2 files changed, 18 insertions(+), 18 deletions(-) diff --git a/services/real-time/app/coffee/WebsocketController.coffee b/services/real-time/app/coffee/WebsocketController.coffee index bd83b0c19a..fedf4888f7 100644 --- a/services/real-time/app/coffee/WebsocketController.coffee +++ b/services/real-time/app/coffee/WebsocketController.coffee @@ -17,7 +17,7 @@ module.exports = WebsocketController = joinProject: (client, user, project_id, callback = (error, project, privilegeLevel, protocolVersion) ->) -> if client.disconnected - metrics.inc('disconnected_join_project') + metrics.inc('editor.join-project.disconnected', 1, {status: 'immediately'}) return callback() user_id = user?._id @@ -26,7 +26,7 @@ module.exports = WebsocketController = WebApiManager.joinProject project_id, user, (error, project, privilegeLevel, isRestrictedUser) -> return callback(error) if error? if client.disconnected - metrics.inc('disconnected_join_project') + metrics.inc('editor.join-project.disconnected', 1, {status: 'after-web-api-call'}) return callback() if !privilegeLevel or privilegeLevel == "" @@ -85,7 +85,7 @@ module.exports = WebsocketController = joinDoc: (client, doc_id, fromVersion = -1, options, callback = (error, doclines, version, ops, ranges) ->) -> if client.disconnected - metrics.inc('disconnected_join_doc') + metrics.inc('editor.join-doc.disconnected', 1, {status: 'immediately'}) return callback() metrics.inc "editor.join-doc" @@ -101,14 +101,14 @@ module.exports = WebsocketController = RoomManager.joinDoc client, doc_id, (error) -> return callback(error) if error? if client.disconnected - metrics.inc('disconnected_join_doc') + metrics.inc('editor.join-doc.disconnected', 1, {status: 'after-joining-room'}) # the client will not read the response anyways return callback() DocumentUpdaterManager.getDocument project_id, doc_id, fromVersion, (error, lines, version, ranges, ops) -> return callback(error) if error? if client.disconnected - metrics.inc('disconnected_join_doc') + metrics.inc('editor.join-doc.disconnected', 1, {status: 'after-doc-updater-call'}) # the client will not read the response anyways return callback() @@ -255,7 +255,7 @@ module.exports = WebsocketController = setTimeout () -> if client.disconnected # skip the message broadcast, the client has moved on - return metrics.inc('disconnected_otUpdateError') + return metrics.inc('editor.doc-update.disconnected', 1, {status:'at-otUpdateError'}) client.emit "otUpdateError", message.error, message client.disconnect() , 100 diff --git a/services/real-time/test/unit/coffee/WebsocketControllerTests.coffee b/services/real-time/test/unit/coffee/WebsocketControllerTests.coffee index 3819b04d7b..8a2ca25c68 100644 --- a/services/real-time/test/unit/coffee/WebsocketControllerTests.coffee +++ b/services/real-time/test/unit/coffee/WebsocketControllerTests.coffee @@ -160,8 +160,8 @@ describe 'WebsocketController', -> it "should call the callback with no details", -> expect(@callback.args[0]).to.deep.equal [] - it "should increment the disconnected_join_project metric", -> - expect(@metrics.inc.calledWith("disconnected_join_project")).to.equal(true) + it "should increment the editor.join-project.disconnected metric with a status", -> + expect(@metrics.inc.calledWith('editor.join-project.disconnected', 1, {status: 'immediately'})).to.equal(true) describe "when the client disconnects while WebApiManager.joinProject is running", -> beforeEach -> @@ -174,8 +174,8 @@ describe 'WebsocketController', -> it "should call the callback with no details", -> expect(@callback.args[0]).to.deep.equal [] - it "should increment the disconnected_join_project metric", -> - expect(@metrics.inc.calledWith("disconnected_join_project")).to.equal(true) + it "should increment the editor.join-project.disconnected metric with a status", -> + expect(@metrics.inc.calledWith('editor.join-project.disconnected', 1, {status: 'after-web-api-call'})).to.equal(true) describe "leaveProject", -> beforeEach -> @@ -422,8 +422,8 @@ describe 'WebsocketController', -> it "should call the callback with no details", -> expect(@callback.args[0]).to.deep.equal([]) - it "should increment the disconnected_join_doc metric", -> - expect(@metrics.inc.calledWith("disconnected_join_doc")).to.equal(true) + it "should increment the editor.join-doc.disconnected metric with a status", -> + expect(@metrics.inc.calledWith('editor.join-doc.disconnected', 1, {status: 'immediately'})).to.equal(true) it "should not get the document", -> expect(@DocumentUpdaterManager.getDocument.called).to.equal(false) @@ -439,8 +439,8 @@ describe 'WebsocketController', -> it "should call the callback with no details", -> expect(@callback.args[0]).to.deep.equal([]) - it "should increment the disconnected_join_doc metric", -> - expect(@metrics.inc.calledWith("disconnected_join_doc")).to.equal(true) + it "should increment the editor.join-doc.disconnected metric with a status", -> + expect(@metrics.inc.calledWith('editor.join-doc.disconnected', 1, {status: 'after-joining-room'})).to.equal(true) it "should not get the document", -> expect(@DocumentUpdaterManager.getDocument.called).to.equal(false) @@ -456,8 +456,8 @@ describe 'WebsocketController', -> it "should call the callback with no details", -> expect(@callback.args[0]).to.deep.equal [] - it "should increment the disconnected_join_doc metric", -> - expect(@metrics.inc.calledWith("disconnected_join_doc")).to.equal(true) + it "should increment the editor.join-doc.disconnected metric with a status", -> + expect(@metrics.inc.calledWith('editor.join-doc.disconnected', 1, {status: 'after-doc-updater-call'})).to.equal(true) describe "leaveDoc", -> beforeEach -> @@ -847,8 +847,8 @@ describe 'WebsocketController', -> it "should not disconnect the client", -> @client.disconnect.called.should.equal false - it "should increment the disconnected_otUpdateError metric", -> - expect(@metrics.inc.calledWith("disconnected_otUpdateError")).to.equal(true) + it "should increment the editor.doc-update.disconnected metric with a status", -> + expect(@metrics.inc.calledWith('editor.doc-update.disconnected', 1, {status:'at-otUpdateError'})).to.equal(true) describe "_assertClientCanApplyUpdate", -> beforeEach -> From f40241a037bf568df3e7772a537df0aa0ae4c9fe Mon Sep 17 00:00:00 2001 From: Jakob Ackermann Date: Fri, 5 Jun 2020 11:38:09 +0100 Subject: [PATCH 343/491] [misc] downgrade logging when running tests --- .../test/acceptance/coffee/helpers/RealtimeServer.coffee | 3 +-- services/real-time/test/unit/coffee/ChannelManagerTests.coffee | 1 + 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/services/real-time/test/acceptance/coffee/helpers/RealtimeServer.coffee b/services/real-time/test/acceptance/coffee/helpers/RealtimeServer.coffee index 12efd1ef13..3a721c18ed 100644 --- a/services/real-time/test/acceptance/coffee/helpers/RealtimeServer.coffee +++ b/services/real-time/test/acceptance/coffee/helpers/RealtimeServer.coffee @@ -1,5 +1,4 @@ app = require('../../../../app') -require("logger-sharelatex").logger.level("info") logger = require("logger-sharelatex") Settings = require("settings-sharelatex") @@ -21,4 +20,4 @@ module.exports = logger.log("clsi running in dev mode") for callback in @callbacks - callback() \ No newline at end of file + callback() diff --git a/services/real-time/test/unit/coffee/ChannelManagerTests.coffee b/services/real-time/test/unit/coffee/ChannelManagerTests.coffee index cb77991f58..354e956283 100644 --- a/services/real-time/test/unit/coffee/ChannelManagerTests.coffee +++ b/services/real-time/test/unit/coffee/ChannelManagerTests.coffee @@ -12,6 +12,7 @@ describe 'ChannelManager', -> @ChannelManager = SandboxedModule.require modulePath, requires: "settings-sharelatex": @settings = {} "metrics-sharelatex": @metrics = {inc: sinon.stub(), summary: sinon.stub()} + "logger-sharelatex": @logger = { log: sinon.stub(), warn: sinon.stub(), error: sinon.stub() } describe "subscribe", -> From 32af7001fc6b9355d37914df52a27388cf615c42 Mon Sep 17 00:00:00 2001 From: Jakob Ackermann Date: Mon, 8 Jun 2020 11:29:40 +0100 Subject: [PATCH 344/491] [misc] Router: prefix the publicId with 'P.' for easy differentiation --- services/real-time/app/coffee/Router.coffee | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/services/real-time/app/coffee/Router.coffee b/services/real-time/app/coffee/Router.coffee index 0266c71eec..91fa27930b 100644 --- a/services/real-time/app/coffee/Router.coffee +++ b/services/real-time/app/coffee/Router.coffee @@ -68,7 +68,7 @@ module.exports = Router = return # send positive confirmation that the client has a valid connection - client.publicId = base64id.generateId() + client.publicId = 'P.' + base64id.generateId() client.emit("connectionAccepted", null, client.publicId) metrics.inc('socket-io.connection') From dc553c4150d465fb42b7cce281d0a78cd717b895 Mon Sep 17 00:00:00 2001 From: Jakob Ackermann Date: Sat, 6 Jun 2020 13:37:40 +0100 Subject: [PATCH 345/491] [misc] vendor a patched session.socket.io middleware --- services/real-time/app.coffee | 2 +- .../app/coffee/SessionSockets.coffee | 23 ++++ services/real-time/package-lock.json | 5 - services/real-time/package.json | 1 - .../coffee/SessionSocketsTests.coffee | 67 ++++++++++ .../unit/coffee/SessionSocketsTests.coffee | 126 ++++++++++++++++++ 6 files changed, 217 insertions(+), 7 deletions(-) create mode 100644 services/real-time/app/coffee/SessionSockets.coffee create mode 100644 services/real-time/test/acceptance/coffee/SessionSocketsTests.coffee create mode 100644 services/real-time/test/unit/coffee/SessionSocketsTests.coffee diff --git a/services/real-time/app.coffee b/services/real-time/app.coffee index 06fd3555f6..9f4c9cc44f 100644 --- a/services/real-time/app.coffee +++ b/services/real-time/app.coffee @@ -17,7 +17,7 @@ if Settings.sentry?.dsn? sessionRedisClient = redis.createClient(Settings.redis.websessions) RedisStore = require('connect-redis')(session) -SessionSockets = require('session.socket.io') +SessionSockets = require('./app/js/SessionSockets') CookieParser = require("cookie-parser") DrainManager = require("./app/js/DrainManager") diff --git a/services/real-time/app/coffee/SessionSockets.coffee b/services/real-time/app/coffee/SessionSockets.coffee new file mode 100644 index 0000000000..229e07b3bb --- /dev/null +++ b/services/real-time/app/coffee/SessionSockets.coffee @@ -0,0 +1,23 @@ +{EventEmitter} = require('events') + +module.exports = (io, sessionStore, cookieParser, cookieName) -> + missingSessionError = new Error('could not look up session by key') + + sessionSockets = new EventEmitter() + next = (error, socket, session) -> + sessionSockets.emit 'connection', error, socket, session + + io.on 'connection', (socket) -> + req = socket.handshake + cookieParser req, {}, () -> + sessionId = req.signedCookies and req.signedCookies[cookieName] + if not sessionId + return next(missingSessionError, socket) + sessionStore.get sessionId, (error, session) -> + if error + return next(error, socket) + if not session + return next(missingSessionError, socket) + next(null, socket, session) + + return sessionSockets diff --git a/services/real-time/package-lock.json b/services/real-time/package-lock.json index 1541d23171..3256a58aca 100644 --- a/services/real-time/package-lock.json +++ b/services/real-time/package-lock.json @@ -2165,11 +2165,6 @@ "send": "0.16.2" } }, - "session.socket.io": { - "version": "0.1.6", - "resolved": "https://registry.npmjs.org/session.socket.io/-/session.socket.io-0.1.6.tgz", - "integrity": "sha1-vh7sJAYJWgP4dw6ozN5s8SYBxdA=" - }, "setprototypeof": { "version": "1.1.0", "resolved": "https://registry.npmjs.org/setprototypeof/-/setprototypeof-1.1.0.tgz", diff --git a/services/real-time/package.json b/services/real-time/package.json index 356a78e712..b50fb9c6ce 100644 --- a/services/real-time/package.json +++ b/services/real-time/package.json @@ -33,7 +33,6 @@ "metrics-sharelatex": "^2.6.2", "redis-sharelatex": "^1.0.12", "request": "^2.88.0", - "session.socket.io": "^0.1.6", "settings-sharelatex": "^1.1.0", "socket.io": "0.9.19", "socket.io-client": "^0.9.16" diff --git a/services/real-time/test/acceptance/coffee/SessionSocketsTests.coffee b/services/real-time/test/acceptance/coffee/SessionSocketsTests.coffee new file mode 100644 index 0000000000..3009da682f --- /dev/null +++ b/services/real-time/test/acceptance/coffee/SessionSocketsTests.coffee @@ -0,0 +1,67 @@ +RealTimeClient = require("./helpers/RealTimeClient") +Settings = require("settings-sharelatex") +{expect} = require('chai') + +describe 'SessionSockets', -> + before -> + @checkSocket = (fn) -> + client = RealTimeClient.connect() + client.on 'connectionAccepted', fn + client.on 'connectionRejected', fn + return null + + describe 'without cookies', -> + before -> + RealTimeClient.cookie = null + + it 'should return a lookup error', (done) -> + @checkSocket (error) -> + expect(error).to.exist + expect(error.message).to.equal('invalid session') + done() + + describe 'with a different cookie', -> + before -> + RealTimeClient.cookie = "some.key=someValue" + + it 'should return a lookup error', (done) -> + @checkSocket (error) -> + expect(error).to.exist + expect(error.message).to.equal('invalid session') + done() + + describe 'with an invalid cookie', -> + before (done) -> + RealTimeClient.setSession {}, (error) -> + return done(error) if error + RealTimeClient.cookie = "#{Settings.cookieName}=#{ + RealTimeClient.cookie.slice(17, 49) + }" + done() + return null + + it 'should return a lookup error', (done) -> + @checkSocket (error) -> + expect(error).to.exist + expect(error.message).to.equal('invalid session') + done() + + describe 'with a valid cookie and no matching session', -> + before -> + RealTimeClient.cookie = "#{Settings.cookieName}=unknownId" + + it 'should return a lookup error', (done) -> + @checkSocket (error) -> + expect(error).to.exist + expect(error.message).to.equal('invalid session') + done() + + describe 'with a valid cookie and a matching session', -> + before (done) -> + RealTimeClient.setSession({}, done) + return null + + it 'should not return an error', (done) -> + @checkSocket (error) -> + expect(error).to.not.exist + done() diff --git a/services/real-time/test/unit/coffee/SessionSocketsTests.coffee b/services/real-time/test/unit/coffee/SessionSocketsTests.coffee new file mode 100644 index 0000000000..2f81699309 --- /dev/null +++ b/services/real-time/test/unit/coffee/SessionSocketsTests.coffee @@ -0,0 +1,126 @@ +{EventEmitter} = require('events') +{expect} = require('chai') +SandboxedModule = require('sandboxed-module') +modulePath = '../../../app/js/SessionSockets' +sinon = require('sinon') + +describe 'SessionSockets', -> + before -> + @SessionSocketsModule = SandboxedModule.require modulePath + @io = new EventEmitter() + @id1 = Math.random().toString() + @id2 = Math.random().toString() + redisResponses = + error: [new Error('Redis: something went wrong'), null] + unknownId: [null, null] + redisResponses[@id1] = [null, {user: {_id: '123'}}] + redisResponses[@id2] = [null, {user: {_id: 'abc'}}] + + @sessionStore = + get: sinon.stub().callsFake (id, fn) -> + fn.apply(null, redisResponses[id]) + @cookieParser = (req, res, next) -> + req.signedCookies = req._signedCookies + next() + @SessionSockets = @SessionSocketsModule(@io, @sessionStore, @cookieParser, 'ol.sid') + @checkSocket = (socket, fn) => + @SessionSockets.once('connection', fn) + @io.emit('connection', socket) + + describe 'without cookies', -> + before -> + @socket = {handshake: {}} + + it 'should return a lookup error', (done) -> + @checkSocket @socket, (error) -> + expect(error).to.exist + expect(error.message).to.equal('could not look up session by key') + done() + + it 'should not query redis', (done) -> + @checkSocket @socket, () => + expect(@sessionStore.get.called).to.equal(false) + done() + + describe 'with a different cookie', -> + before -> + @socket = {handshake: {_signedCookies: {other: 1}}} + + it 'should return a lookup error', (done) -> + @checkSocket @socket, (error) -> + expect(error).to.exist + expect(error.message).to.equal('could not look up session by key') + done() + + it 'should not query redis', (done) -> + @checkSocket @socket, () => + expect(@sessionStore.get.called).to.equal(false) + done() + + describe 'with a valid cookie and a failing session lookup', -> + before -> + @socket = {handshake: {_signedCookies: {'ol.sid': 'error'}}} + + it 'should query redis', (done) -> + @checkSocket @socket, () => + expect(@sessionStore.get.called).to.equal(true) + done() + + it 'should return a redis error', (done) -> + @checkSocket @socket, (error) -> + expect(error).to.exist + expect(error.message).to.equal('Redis: something went wrong') + done() + + describe 'with a valid cookie and no matching session', -> + before -> + @socket = {handshake: {_signedCookies: {'ol.sid': 'unknownId'}}} + + it 'should query redis', (done) -> + @checkSocket @socket, () => + expect(@sessionStore.get.called).to.equal(true) + done() + + it 'should return a lookup error', (done) -> + @checkSocket @socket, (error) -> + expect(error).to.exist + expect(error.message).to.equal('could not look up session by key') + done() + + describe 'with a valid cookie and a matching session', -> + before -> + @socket = {handshake: {_signedCookies: {'ol.sid': @id1}}} + + it 'should query redis', (done) -> + @checkSocket @socket, () => + expect(@sessionStore.get.called).to.equal(true) + done() + + it 'should not return an error', (done) -> + @checkSocket @socket, (error) -> + expect(error).to.not.exist + done() + + it 'should return the session', (done) -> + @checkSocket @socket, (error, s, session) -> + expect(session).to.deep.equal({user: {_id: '123'}}) + done() + + describe 'with a different valid cookie and matching session', -> + before -> + @socket = {handshake: {_signedCookies: {'ol.sid': @id2}}} + + it 'should query redis', (done) -> + @checkSocket @socket, () => + expect(@sessionStore.get.called).to.equal(true) + done() + + it 'should not return an error', (done) -> + @checkSocket @socket, (error) -> + expect(error).to.not.exist + done() + + it 'should return the other session', (done) -> + @checkSocket @socket, (error, s, session) -> + expect(session).to.deep.equal({user: {_id: 'abc'}}) + done() From 853ee994a68d92a44bbb9bb54e5b9e1a189b7242 Mon Sep 17 00:00:00 2001 From: Jakob Ackermann Date: Mon, 18 May 2020 18:25:27 +0200 Subject: [PATCH 346/491] [perf] add a few short cuts to the packet decoding --- services/real-time/socket.io.patch.js | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/services/real-time/socket.io.patch.js b/services/real-time/socket.io.patch.js index 796f8c7574..354d8223c3 100644 --- a/services/real-time/socket.io.patch.js +++ b/services/real-time/socket.io.patch.js @@ -47,3 +47,13 @@ function patchedFrameHandler(opcode, str) { } return outputBuffer; } + +const parser = require('socket.io/lib/parser') +const decodePacket = parser.decodePacket +parser.decodePacket = function (data) { + if (typeof data !== 'string') return {} + const firstColon = data.indexOf(':') + if (firstColon === -1) return {} + if (data.indexOf(':', firstColon + 1) === -1) return {} + return decodePacket(data) +} From acb7d7df5a85db3fc6b88b4d00be9215dfcef8ec Mon Sep 17 00:00:00 2001 From: Jakob Ackermann Date: Tue, 9 Jun 2020 16:30:03 +0100 Subject: [PATCH 347/491] [misc] add test cases for the validation of the callback argument When the user provides a function as last argument for socket.emit, socket.io will flag this as an RPC and add a cb as the last argument to the client.on('event', ...) handler on the server side. Without a function as last argument for socket.emit, the callback argument on the server side is undefined, leading to invalid function calls (`undefined()`) and an unhandled exception. The user can also provide lots of other arguments, so the 2nd/3rd ... argument is of arbitrary type, again leading to invalid function calls -- e.g. `1()`. --- services/real-time/app/coffee/Router.coffee | 1 + .../test/acceptance/coffee/RouterTests.coffee | 76 +++++++++++++++++++ 2 files changed, 77 insertions(+) create mode 100644 services/real-time/test/acceptance/coffee/RouterTests.coffee diff --git a/services/real-time/app/coffee/Router.coffee b/services/real-time/app/coffee/Router.coffee index 276c64e2dd..93cdd07926 100644 --- a/services/real-time/app/coffee/Router.coffee +++ b/services/real-time/app/coffee/Router.coffee @@ -26,6 +26,7 @@ module.exports = Router = logger.warn attrs, error.message, code: error.code return callback {message: error.message, code: error.code} if error.message == 'unexpected arguments' + # the payload might be very large, put it on level info logger.log attrs, 'unexpected arguments' metrics.inc 'unexpected-arguments', 1, { status: method } return callback { message: error.message } diff --git a/services/real-time/test/acceptance/coffee/RouterTests.coffee b/services/real-time/test/acceptance/coffee/RouterTests.coffee new file mode 100644 index 0000000000..c3952a2887 --- /dev/null +++ b/services/real-time/test/acceptance/coffee/RouterTests.coffee @@ -0,0 +1,76 @@ +async = require "async" +{expect} = require("chai") + +RealTimeClient = require "./helpers/RealTimeClient" +FixturesManager = require "./helpers/FixturesManager" + + +describe "Router", -> + describe "joinProject", -> + describe "when there is no callback provided", -> + after () -> + process.removeListener('unhandledRejection', @onUnhandled) + + before (done) -> + @onUnhandled = (error) -> + done(error) + process.on('unhandledRejection', @onUnhandled) + async.series [ + (cb) => + FixturesManager.setUpProject { + privilegeLevel: "owner" + project: { + name: "Test Project" + } + }, (e, {@project_id, @user_id}) => + cb(e) + + (cb) => + @client = RealTimeClient.connect() + @client.on "connectionAccepted", cb + + (cb) => + @client = RealTimeClient.connect() + @client.on "connectionAccepted", cb + + (cb) => + @client.emit "joinProject", project_id: @project_id + setTimeout(cb, 100) + ], done + + it "should keep on going", -> + expect('still running').to.exist + + describe "when there are too many arguments", -> + after () -> + process.removeListener('unhandledRejection', @onUnhandled) + + before (done) -> + @onUnhandled = (error) -> + done(error) + process.on('unhandledRejection', @onUnhandled) + async.series [ + (cb) => + FixturesManager.setUpProject { + privilegeLevel: "owner" + project: { + name: "Test Project" + } + }, (e, {@project_id, @user_id}) => + cb(e) + + (cb) => + @client = RealTimeClient.connect() + @client.on "connectionAccepted", cb + + (cb) => + @client = RealTimeClient.connect() + @client.on "connectionAccepted", cb + + (cb) => + @client.emit "joinProject", 1, 2, 3, 4, 5, (@error) => + cb() + ], done + + it "should return an error message", -> + expect(@error.message).to.equal('unexpected arguments') From 91e296533fa65b9a3779fabdc36c427ead6a41d7 Mon Sep 17 00:00:00 2001 From: Jakob Ackermann Date: Fri, 21 Feb 2020 15:11:09 +0000 Subject: [PATCH 348/491] [misc] test/acceptance: add tests for the draining of connections --- .../coffee/DrainManagerTests.coffee | 72 +++++++++++++++++++ 1 file changed, 72 insertions(+) create mode 100644 services/real-time/test/acceptance/coffee/DrainManagerTests.coffee diff --git a/services/real-time/test/acceptance/coffee/DrainManagerTests.coffee b/services/real-time/test/acceptance/coffee/DrainManagerTests.coffee new file mode 100644 index 0000000000..cffae6fda6 --- /dev/null +++ b/services/real-time/test/acceptance/coffee/DrainManagerTests.coffee @@ -0,0 +1,72 @@ +RealTimeClient = require "./helpers/RealTimeClient" +FixturesManager = require "./helpers/FixturesManager" + +expect = require("chai").expect + +async = require "async" +request = require "request" + +Settings = require "settings-sharelatex" + +drain = (rate, callback) -> + request.post { + url: "http://localhost:3026/drain?rate=#{rate}" + auth: { + user: Settings.internal.realTime.user, + pass: Settings.internal.realTime.pass + } + }, (error, response, data) -> + callback error, data + return null + +describe "DrainManagerTests", -> + before (done) -> + FixturesManager.setUpProject { + privilegeLevel: "owner" + project: { + name: "Test Project" + } + }, (e, {@project_id, @user_id}) => done() + return null + + describe "with two clients in the project", -> + beforeEach (done) -> + async.series [ + (cb) => + @clientA = RealTimeClient.connect() + @clientA.on "connectionAccepted", cb + + (cb) => + @clientB = RealTimeClient.connect() + @clientB.on "connectionAccepted", cb + + (cb) => + @clientA.emit "joinProject", project_id: @project_id, cb + + (cb) => + @clientB.emit "joinProject", project_id: @project_id, cb + ], done + + describe "starting to drain", () -> + # there is an internal delay of 1000ms, shift accordingly + @timeout(5000) + beforeEach (done) -> + async.parallel [ + (cb) => + @clientA.on "reconnectGracefully", cb + (cb) => + @clientB.on "reconnectGracefully", cb + + (cb) -> drain(2, cb) + ], done + + afterEach (done) -> + # reset drain + drain(0, done) + + it "should not timeout", -> + expect(true).to.equal(true) + + it "should not have disconnected", -> + expect(@clientA.socket.connected).to.equal true + expect(@clientB.socket.connected).to.equal true From de35fc5ecf6e91b8e0e39a0e3e3020428def4718 Mon Sep 17 00:00:00 2001 From: Jakob Ackermann Date: Mon, 24 Feb 2020 13:28:22 +0100 Subject: [PATCH 349/491] [HttpApiController] implement the disconnection of a single client The http route returns as soon as the client has fully disconnected. --- .../app/coffee/HttpApiController.coffee | 16 +++++++++++++++- services/real-time/app/coffee/Router.coffee | 1 + 2 files changed, 16 insertions(+), 1 deletion(-) diff --git a/services/real-time/app/coffee/HttpApiController.coffee b/services/real-time/app/coffee/HttpApiController.coffee index f99bef6dd6..299d198f57 100644 --- a/services/real-time/app/coffee/HttpApiController.coffee +++ b/services/real-time/app/coffee/HttpApiController.coffee @@ -18,4 +18,18 @@ module.exports = HttpApiController = rate = parseFloat(rate) || 0 logger.log {rate}, "setting client drain rate" DrainManager.startDrain io, rate - res.send 204 \ No newline at end of file + res.send 204 + + disconnectClient: (req, res, next) -> + io = req.app.get("io") + client_id = req.params.client_id + client = io.sockets.sockets[client_id] + + if !client + logger.info({client_id}, "api: client already disconnected") + res.sendStatus(404) + return + logger.warn({client_id}, "api: requesting client disconnect") + client.on "disconnect", () -> + res.sendStatus(204) + client.disconnect() diff --git a/services/real-time/app/coffee/Router.coffee b/services/real-time/app/coffee/Router.coffee index 276c64e2dd..99aacce3bc 100644 --- a/services/real-time/app/coffee/Router.coffee +++ b/services/real-time/app/coffee/Router.coffee @@ -53,6 +53,7 @@ module.exports = Router = app.post "/project/:project_id/message/:message", httpAuth, bodyParser.json(limit: "5mb"), HttpApiController.sendMessage app.post "/drain", httpAuth, HttpApiController.startDrain + app.post "/client/:client_id/disconnect", httpAuth, HttpApiController.disconnectClient session.on 'connection', (error, client, session) -> client?.on "error", (err) -> From eabff1d6b2f777f5fa2a2e0077b13aa0bdca6486 Mon Sep 17 00:00:00 2001 From: Jakob Ackermann Date: Mon, 24 Feb 2020 12:30:24 +0000 Subject: [PATCH 350/491] [perf] test/acceptance: DrainManagerTests: cleanup previous clients ...before starting to drain. --- .../acceptance/coffee/DrainManagerTests.coffee | 14 ++++++++++++-- .../coffee/helpers/RealTimeClient.coffee | 18 ++++++++++++++++++ 2 files changed, 30 insertions(+), 2 deletions(-) diff --git a/services/real-time/test/acceptance/coffee/DrainManagerTests.coffee b/services/real-time/test/acceptance/coffee/DrainManagerTests.coffee index cffae6fda6..b5b192cf88 100644 --- a/services/real-time/test/acceptance/coffee/DrainManagerTests.coffee +++ b/services/real-time/test/acceptance/coffee/DrainManagerTests.coffee @@ -29,6 +29,18 @@ describe "DrainManagerTests", -> }, (e, {@project_id, @user_id}) => done() return null + before (done) -> + # cleanup to speedup reconnecting + @timeout(10000) + RealTimeClient.disconnectAllClients done + + # trigger and check cleanup + it "should have disconnected all previous clients", (done) -> + RealTimeClient.getConnectedClients (error, data) -> + return done(error) if error + expect(data.length).to.equal(0) + done() + describe "with two clients in the project", -> beforeEach (done) -> async.series [ @@ -48,8 +60,6 @@ describe "DrainManagerTests", -> ], done describe "starting to drain", () -> - # there is an internal delay of 1000ms, shift accordingly - @timeout(5000) beforeEach (done) -> async.parallel [ (cb) => diff --git a/services/real-time/test/acceptance/coffee/helpers/RealTimeClient.coffee b/services/real-time/test/acceptance/coffee/helpers/RealTimeClient.coffee index e3254a79a5..7d54e23b3c 100644 --- a/services/real-time/test/acceptance/coffee/helpers/RealTimeClient.coffee +++ b/services/real-time/test/acceptance/coffee/helpers/RealTimeClient.coffee @@ -1,5 +1,6 @@ XMLHttpRequest = require("../../libs/XMLHttpRequest").XMLHttpRequest io = require("socket.io-client") +async = require("async") request = require "request" Settings = require "settings-sharelatex" @@ -55,3 +56,20 @@ module.exports = Client = }, (error, response, data) -> callback error, data + + disconnectClient: (client_id, callback) -> + request.post { + url: "http://localhost:3026/client/#{client_id}/disconnect" + auth: { + user: Settings.internal.realTime.user, + pass: Settings.internal.realTime.pass + } + }, (error, response, data) -> + callback error, data + return null + + disconnectAllClients: (callback) -> + Client.getConnectedClients (error, clients) -> + async.each clients, (clientView, cb) -> + Client.disconnectClient clientView.client_id, cb + , callback From bc4449446621929b23dc29f2306ca6adf183bdc5 Mon Sep 17 00:00:00 2001 From: Jakob Ackermann Date: Mon, 24 Feb 2020 11:07:06 +0000 Subject: [PATCH 351/491] [HttpController] return 404 in case of a missing client and add tests Add acceptance tests for the client view. --- .../app/coffee/HttpController.coffee | 6 +- .../coffee/HttpControllerTests.coffee | 68 +++++++++++++++++++ 2 files changed, 72 insertions(+), 2 deletions(-) create mode 100644 services/real-time/test/acceptance/coffee/HttpControllerTests.coffee diff --git a/services/real-time/app/coffee/HttpController.coffee b/services/real-time/app/coffee/HttpController.coffee index 91f8df0df8..ae92a9e299 100644 --- a/services/real-time/app/coffee/HttpController.coffee +++ b/services/real-time/app/coffee/HttpController.coffee @@ -30,8 +30,10 @@ module.exports = HttpController = getConnectedClient: (req, res, next) -> {client_id} = req.params io = req.app.get("io") - ioClient = io.sockets.socket(client_id) + ioClient = io.sockets.sockets[client_id] + if !ioClient + res.sendStatus(404) + return HttpController._getConnectedClientView ioClient, (error, client) -> return next(error) if error? res.json client - \ No newline at end of file diff --git a/services/real-time/test/acceptance/coffee/HttpControllerTests.coffee b/services/real-time/test/acceptance/coffee/HttpControllerTests.coffee new file mode 100644 index 0000000000..524ea7e5de --- /dev/null +++ b/services/real-time/test/acceptance/coffee/HttpControllerTests.coffee @@ -0,0 +1,68 @@ +async = require('async') +expect = require('chai').expect +request = require('request').defaults({ + baseUrl: 'http://localhost:3026' +}) + +RealTimeClient = require "./helpers/RealTimeClient" +FixturesManager = require "./helpers/FixturesManager" + +describe 'HttpControllerTests', -> + describe 'without a user', -> + it 'should return 404 for the client view', (done) -> + client_id = 'not-existing' + request.get { + url: "/clients/#{client_id}" + json: true + }, (error, response, data) -> + return done(error) if error + expect(response.statusCode).to.equal(404) + done() + + describe 'with a user and after joining a project', -> + before (done) -> + async.series [ + (cb) => + FixturesManager.setUpProject { + privilegeLevel: "owner" + }, (error, {@project_id, @user_id}) => + cb(error) + + (cb) => + FixturesManager.setUpDoc @project_id, {}, (error, {@doc_id}) => + cb(error) + + (cb) => + @client = RealTimeClient.connect() + @client.on "connectionAccepted", cb + + (cb) => + @client.emit "joinProject", {@project_id}, cb + + (cb) => + @client.emit "joinDoc", @doc_id, cb + ], done + + it 'should send a client view', (done) -> + request.get { + url: "/clients/#{@client.socket.sessionid}" + json: true + }, (error, response, data) => + return done(error) if error + expect(response.statusCode).to.equal(200) + expect(data.connected_time).to.exist + delete data.connected_time + # .email is not set in the session + delete data.email + expect(data).to.deep.equal({ + client_id: @client.socket.sessionid, + first_name: 'Joe', + last_name: 'Bloggs', + project_id: @project_id, + user_id: @user_id, + rooms: [ + @project_id, + @doc_id, + ] + }) + done() From 56fda1f9b00f31e98a64fe5f37e81fc35ceac765 Mon Sep 17 00:00:00 2001 From: Jakob Ackermann Date: Fri, 21 Feb 2020 13:59:23 +0000 Subject: [PATCH 352/491] [misc] test/acceptance: use the correct redis instances --- .../real-time/test/acceptance/coffee/ApplyUpdateTests.coffee | 2 +- .../real-time/test/acceptance/coffee/ReceiveUpdateTests.coffee | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/services/real-time/test/acceptance/coffee/ApplyUpdateTests.coffee b/services/real-time/test/acceptance/coffee/ApplyUpdateTests.coffee index e7babeb81c..f2437f2641 100644 --- a/services/real-time/test/acceptance/coffee/ApplyUpdateTests.coffee +++ b/services/real-time/test/acceptance/coffee/ApplyUpdateTests.coffee @@ -8,7 +8,7 @@ FixturesManager = require "./helpers/FixturesManager" settings = require "settings-sharelatex" redis = require "redis-sharelatex" -rclient = redis.createClient(settings.redis.websessions) +rclient = redis.createClient(settings.redis.documentupdater) redisSettings = settings.redis diff --git a/services/real-time/test/acceptance/coffee/ReceiveUpdateTests.coffee b/services/real-time/test/acceptance/coffee/ReceiveUpdateTests.coffee index fb504dd9aa..7f04fd8adb 100644 --- a/services/real-time/test/acceptance/coffee/ReceiveUpdateTests.coffee +++ b/services/real-time/test/acceptance/coffee/ReceiveUpdateTests.coffee @@ -10,7 +10,7 @@ async = require "async" settings = require "settings-sharelatex" redis = require "redis-sharelatex" -rclient = redis.createClient(settings.redis.websessions) +rclient = redis.createClient(settings.redis.pubsub) describe "receiveUpdate", -> before (done) -> From 83e3ff0ed7dc04874b7fbdd5463877bf05d17da6 Mon Sep 17 00:00:00 2001 From: Jakob Ackermann Date: Fri, 21 Feb 2020 14:13:30 +0000 Subject: [PATCH 353/491] [misc] test/acceptance: ReceiveUpdateTests: add 2nd project/3rd client ...and check for cross project leakage. --- .../coffee/ReceiveUpdateTests.coffee | 125 +++++++++++++++--- 1 file changed, 105 insertions(+), 20 deletions(-) diff --git a/services/real-time/test/acceptance/coffee/ReceiveUpdateTests.coffee b/services/real-time/test/acceptance/coffee/ReceiveUpdateTests.coffee index 7f04fd8adb..4da68b76c5 100644 --- a/services/real-time/test/acceptance/coffee/ReceiveUpdateTests.coffee +++ b/services/real-time/test/acceptance/coffee/ReceiveUpdateTests.coffee @@ -13,7 +13,7 @@ redis = require "redis-sharelatex" rclient = redis.createClient(settings.redis.pubsub) describe "receiveUpdate", -> - before (done) -> + beforeEach (done) -> @lines = ["test", "doc", "lines"] @version = 42 @ops = ["mock", "doc", "ops"] @@ -52,15 +52,52 @@ describe "receiveUpdate", -> (cb) => @clientB.emit "joinDoc", @doc_id, cb + + (cb) => + FixturesManager.setUpProject { + privilegeLevel: "owner" + project: {name: "Test Project"} + }, (error, {user_id: @user_id_second, project_id: @project_id_second}) => cb() + + (cb) => + FixturesManager.setUpDoc @project_id_second, {@lines, @version, @ops}, (e, {doc_id: @doc_id_second}) => + cb(e) + + (cb) => + @clientC = RealTimeClient.connect() + @clientC.on "connectionAccepted", cb + + (cb) => + @clientC.emit "joinProject", { + project_id: @project_id_second + }, cb + (cb) => + @clientC.emit "joinDoc", @doc_id_second, cb + + (cb) => + @clientAUpdates = [] + @clientA.on "otUpdateApplied", (update) => @clientAUpdates.push(update) + @clientBUpdates = [] + @clientB.on "otUpdateApplied", (update) => @clientBUpdates.push(update) + @clientCUpdates = [] + @clientC.on "otUpdateApplied", (update) => @clientCUpdates.push(update) + + @clientAErrors = [] + @clientA.on "otUpdateError", (error) => @clientAErrors.push(error) + @clientBErrors = [] + @clientB.on "otUpdateError", (error) => @clientBErrors.push(error) + @clientCErrors = [] + @clientC.on "otUpdateError", (error) => @clientCErrors.push(error) + cb() ], done - + + afterEach () -> + @clientA?.disconnect() + @clientB?.disconnect() + @clientC?.disconnect() + describe "with an update from clientA", -> - before (done) -> - @clientAUpdates = [] - @clientA.on "otUpdateApplied", (update) => @clientAUpdates.push(update) - @clientBUpdates = [] - @clientB.on "otUpdateApplied", (update) => @clientBUpdates.push(update) - + beforeEach (done) -> @update = { doc_id: @doc_id op: @@ -80,21 +117,69 @@ describe "receiveUpdate", -> @clientAUpdates.should.deep.equal [{ v: @version, doc: @doc_id }] - - describe "with an error", -> - before (done) -> - @clientAErrors = [] - @clientA.on "otUpdateError", (error) => @clientAErrors.push(error) - @clientBErrors = [] - @clientB.on "otUpdateError", (error) => @clientBErrors.push(error) - + + it "should send nothing to clientC", -> + @clientCUpdates.should.deep.equal [] + + describe "with an update from clientC", -> + beforeEach (done) -> + @update = { + doc_id: @doc_id_second + op: + meta: + source: @clientC.publicId + v: @version + doc: @doc_id_second + op: [{i: "update from clientC", p: 50}] + } + rclient.publish "applied-ops", JSON.stringify(@update) + setTimeout done, 200 # Give clients time to get message + + it "should send nothing to clientA", -> + @clientAUpdates.should.deep.equal [] + + it "should send nothing to clientB", -> + @clientBUpdates.should.deep.equal [] + + it "should send an ack to clientC", -> + @clientCUpdates.should.deep.equal [{ + v: @version, doc: @doc_id_second + }] + + describe "with an error for the first project", -> + beforeEach (done) -> rclient.publish "applied-ops", JSON.stringify({doc_id: @doc_id, error: @error = "something went wrong"}) setTimeout done, 200 # Give clients time to get message - - it "should send the error to both clients", -> + + it "should send the error to the clients in the first project", -> @clientAErrors.should.deep.equal [@error] @clientBErrors.should.deep.equal [@error] - - it "should disconnect the clients", -> + + it "should not send any errors to the client in the second project", -> + @clientCErrors.should.deep.equal [] + + it "should disconnect the clients of the first project", -> @clientA.socket.connected.should.equal false @clientB.socket.connected.should.equal false + + it "should not disconnect the client in the second project", -> + @clientC.socket.connected.should.equal true + + describe "with an error for the second project", -> + beforeEach (done) -> + rclient.publish "applied-ops", JSON.stringify({doc_id: @doc_id_second, error: @error = "something went wrong"}) + setTimeout done, 200 # Give clients time to get message + + it "should not send any errors to the clients in the first project", -> + @clientAErrors.should.deep.equal [] + @clientBErrors.should.deep.equal [] + + it "should send the error to the client in the second project", -> + @clientCErrors.should.deep.equal [@error] + + it "should not disconnect the clients of the first project", -> + @clientA.socket.connected.should.equal true + @clientB.socket.connected.should.equal true + + it "should disconnect the client in the second project", -> + @clientC.socket.connected.should.equal false From b2e4448992512edc7e10cd88e6bbf3796548c807 Mon Sep 17 00:00:00 2001 From: Jakob Ackermann Date: Wed, 26 Feb 2020 17:56:57 +0100 Subject: [PATCH 354/491] [misc] test/acceptance: ReceiveUpdateTests: test remotely sent update --- .../coffee/ReceiveUpdateTests.coffee | 23 +++++++++++++++++++ 1 file changed, 23 insertions(+) diff --git a/services/real-time/test/acceptance/coffee/ReceiveUpdateTests.coffee b/services/real-time/test/acceptance/coffee/ReceiveUpdateTests.coffee index 4da68b76c5..da9ee0ca36 100644 --- a/services/real-time/test/acceptance/coffee/ReceiveUpdateTests.coffee +++ b/services/real-time/test/acceptance/coffee/ReceiveUpdateTests.coffee @@ -146,6 +146,29 @@ describe "receiveUpdate", -> v: @version, doc: @doc_id_second }] + describe "with an update from a remote client for project 1", -> + beforeEach (done) -> + @update = { + doc_id: @doc_id + op: + meta: + source: 'this-is-a-remote-client-id' + v: @version + doc: @doc_id + op: [{i: "foo", p: 50}] + } + rclient.publish "applied-ops", JSON.stringify(@update) + setTimeout done, 200 # Give clients time to get message + + it "should send the full op to clientA", -> + @clientAUpdates.should.deep.equal [@update.op] + + it "should send the full op to clientB", -> + @clientBUpdates.should.deep.equal [@update.op] + + it "should send nothing to clientC", -> + @clientCUpdates.should.deep.equal [] + describe "with an error for the first project", -> beforeEach (done) -> rclient.publish "applied-ops", JSON.stringify({doc_id: @doc_id, error: @error = "something went wrong"}) From 5f7841526f20f9ea4f7f617bd4ecbf30e3a45872 Mon Sep 17 00:00:00 2001 From: Jakob Ackermann Date: Wed, 17 Jun 2020 09:29:12 +0100 Subject: [PATCH 355/491] [misc] RoomManager: emitOnCompletion: properly handle Promise rejections ``` result = Promise.all([]) # rejection 1 result.then () -> RoomEvents.emit(eventName) # rejection 2 result.catch (err) -> RoomEvents.emit(eventName, err) # handle r1 ``` As shown above, the second rejection remains unhandled. The fix is to chain the `.catch()` onto the `.then()` Promise. --- .../real-time/app/coffee/RoomManager.coffee | 6 ++--- .../test/unit/coffee/RoomManagerTests.coffee | 24 +++++++++++++++++++ 2 files changed, 27 insertions(+), 3 deletions(-) diff --git a/services/real-time/app/coffee/RoomManager.coffee b/services/real-time/app/coffee/RoomManager.coffee index cad3fd0a12..25684ed558 100644 --- a/services/real-time/app/coffee/RoomManager.coffee +++ b/services/real-time/app/coffee/RoomManager.coffee @@ -39,9 +39,9 @@ module.exports = RoomManager = @leaveEntity client, entity, id emitOnCompletion: (promiseList, eventName) -> - result = Promise.all(promiseList) - result.then () -> RoomEvents.emit(eventName) - result.catch (err) -> RoomEvents.emit(eventName, err) + Promise.all(promiseList) + .then(() -> RoomEvents.emit(eventName)) + .catch((err) -> RoomEvents.emit(eventName, err)) eventSource: () -> return RoomEvents diff --git a/services/real-time/test/unit/coffee/RoomManagerTests.coffee b/services/real-time/test/unit/coffee/RoomManagerTests.coffee index fc8375ae54..c81663576d 100644 --- a/services/real-time/test/unit/coffee/RoomManagerTests.coffee +++ b/services/real-time/test/unit/coffee/RoomManagerTests.coffee @@ -1,4 +1,5 @@ chai = require('chai') +expect = chai.expect should = chai.should() sinon = require("sinon") modulePath = "../../../app/js/RoomManager.js" @@ -20,6 +21,29 @@ describe 'RoomManager', -> sinon.spy(@RoomEvents, 'emit') sinon.spy(@RoomEvents, 'once') + describe "emitOnCompletion", -> + describe "when a subscribe errors", -> + afterEach () -> + process.removeListener("unhandledRejection", @onUnhandled) + + beforeEach (done) -> + @onUnhandled = (error) => + @unhandledError = error + done(new Error("unhandledRejection: #{error.message}")) + process.on("unhandledRejection", @onUnhandled) + + reject = undefined + subscribePromise = new Promise((_, r) -> reject = r) + promises = [subscribePromise] + eventName = "project-subscribed-123" + @RoomEvents.once eventName, () -> + setTimeout(done, 100) + @RoomManager.emitOnCompletion(promises, eventName) + setTimeout(() -> reject(new Error("subscribe failed"))) + + it "should keep going", () -> + expect(@unhandledError).to.not.exist + describe "joinProject", -> describe "when the project room is empty", -> From 1e1f6ca19f3a59ccf58b381caac582ef5e51ad26 Mon Sep 17 00:00:00 2001 From: Miguel Serrano Date: Fri, 19 Jun 2020 16:26:32 +0200 Subject: [PATCH 356/491] Updated 'uid-safe' and minor/patch dependencies --- services/real-time/package-lock.json | 803 ++++++++++++++------------- services/real-time/package.json | 16 +- 2 files changed, 423 insertions(+), 396 deletions(-) diff --git a/services/real-time/package-lock.json b/services/real-time/package-lock.json index 3256a58aca..94141c4e6c 100644 --- a/services/real-time/package-lock.json +++ b/services/real-time/package-lock.json @@ -7,7 +7,7 @@ "@google-cloud/common": { "version": "0.32.1", "resolved": "https://registry.npmjs.org/@google-cloud/common/-/common-0.32.1.tgz", - "integrity": "sha512-bLdPzFvvBMtVkwsoBtygE9oUm3yrNmPa71gvOgucYI/GqvNP2tb6RYsDHPq98kvignhcgHGDI5wyNgxaCo8bKQ==", + "integrity": "sha1-ajLDQBcs6j22Z00ODjTnh0CgBz8=", "requires": { "@google-cloud/projectify": "^0.3.3", "@google-cloud/promisify": "^0.4.0", @@ -25,7 +25,7 @@ "@google-cloud/debug-agent": { "version": "3.2.0", "resolved": "https://registry.npmjs.org/@google-cloud/debug-agent/-/debug-agent-3.2.0.tgz", - "integrity": "sha512-fP87kYbS6aeDna08BivwQ1J260mwJGchRi99XdWCgqbRwuFac8ul0OT5i2wEeDSc5QaDX8ZuWQQ0igZvh1rTyQ==", + "integrity": "sha1-2qdjWhaYpWY31dxXzhED536uKdM=", "requires": { "@google-cloud/common": "^0.32.0", "@sindresorhus/is": "^0.15.0", @@ -53,7 +53,7 @@ "@google-cloud/profiler": { "version": "0.2.3", "resolved": "https://registry.npmjs.org/@google-cloud/profiler/-/profiler-0.2.3.tgz", - "integrity": "sha512-rNvtrFtIebIxZEJ/O0t8n7HciZGIXBo8DvHxWqAmsCaeLvkTtsaL6HmPkwxrNQ1IhbYWAxF+E/DwCiHyhKmgTg==", + "integrity": "sha1-Fj3738Mwuug1X+RuHlvgZTV7H1w=", "requires": { "@google-cloud/common": "^0.26.0", "@types/console-log-level": "^1.4.0", @@ -75,7 +75,7 @@ "@google-cloud/common": { "version": "0.26.2", "resolved": "https://registry.npmjs.org/@google-cloud/common/-/common-0.26.2.tgz", - "integrity": "sha512-xJ2M/q3MrUbnYZuFlpF01caAlEhAUoRn0NXp93Hn3pkFpfSOG8YfbKbpBAHvcKVbBOAKVIwPsleNtuyuabUwLQ==", + "integrity": "sha1-nFTiRxqEqgMelaJIJJduCA8lVkU=", "requires": { "@google-cloud/projectify": "^0.3.2", "@google-cloud/promisify": "^0.3.0", @@ -94,7 +94,7 @@ "@google-cloud/promisify": { "version": "0.3.1", "resolved": "https://registry.npmjs.org/@google-cloud/promisify/-/promisify-0.3.1.tgz", - "integrity": "sha512-QzB0/IMvB0eFxFK7Eqh+bfC8NLv3E9ScjWQrPOk6GgfNroxcVITdTlT8NRsRrcp5+QQJVPLkRqKG0PUdaWXmHw==" + "integrity": "sha1-9kHm2USo4KBe4MsQkd+mAIm+zbo=" }, "arrify": { "version": "1.0.1", @@ -104,7 +104,7 @@ "gcp-metadata": { "version": "0.9.3", "resolved": "https://registry.npmjs.org/gcp-metadata/-/gcp-metadata-0.9.3.tgz", - "integrity": "sha512-caV4S84xAjENtpezLCT/GILEAF5h/bC4cNqZFmt/tjTn8t+JBtTkQrgBrJu3857YdsnlM8rxX/PMcKGtE8hUlw==", + "integrity": "sha1-H510lfdGChRSZIHynhFZbdVj3SY=", "requires": { "gaxios": "^1.0.2", "json-bigint": "^0.3.0" @@ -113,7 +113,7 @@ "google-auth-library": { "version": "2.0.2", "resolved": "https://registry.npmjs.org/google-auth-library/-/google-auth-library-2.0.2.tgz", - "integrity": "sha512-FURxmo1hBVmcfLauuMRKOPYAPKht3dGuI2wjeJFalDUThO0HoYVjr4yxt5cgYSFm1dgUpmN9G/poa7ceTFAIiA==", + "integrity": "sha1-ejFdIDZ0Svavyth7IQ7mY4tA9Xs=", "requires": { "axios": "^0.18.0", "gcp-metadata": "^0.7.0", @@ -127,7 +127,7 @@ "gcp-metadata": { "version": "0.7.0", "resolved": "https://registry.npmjs.org/gcp-metadata/-/gcp-metadata-0.7.0.tgz", - "integrity": "sha512-ffjC09amcDWjh3VZdkDngIo7WoluyC5Ag9PAYxZbmQLOLNI8lvPtoKTSCyU54j2gwy5roZh6sSMTfkY2ct7K3g==", + "integrity": "sha1-bDXbtSvaMqQnu5yY9UI33dG1QG8=", "requires": { "axios": "^0.18.0", "extend": "^3.0.1", @@ -146,17 +146,17 @@ "@google-cloud/projectify": { "version": "0.3.3", "resolved": "https://registry.npmjs.org/@google-cloud/projectify/-/projectify-0.3.3.tgz", - "integrity": "sha512-7522YHQ4IhaafgSunsFF15nG0TGVmxgXidy9cITMe+256RgqfcrfWphiMufW+Ou4kqagW/u3yxwbzVEW3dk2Uw==" + "integrity": "sha1-vekQPVCyCj6jM334xng6dm5w1B0=" }, "@google-cloud/promisify": { "version": "0.4.0", "resolved": "https://registry.npmjs.org/@google-cloud/promisify/-/promisify-0.4.0.tgz", - "integrity": "sha512-4yAHDC52TEMCNcMzVC8WlqnKKKq+Ssi2lXoUg9zWWkZ6U6tq9ZBRYLHHCRdfU+EU9YJsVmivwGcKYCjRGjnf4Q==" + "integrity": "sha1-T7/PTYW7ai5MzwWqY9KxDWyarZs=" }, "@google-cloud/trace-agent": { "version": "3.6.1", "resolved": "https://registry.npmjs.org/@google-cloud/trace-agent/-/trace-agent-3.6.1.tgz", - "integrity": "sha512-KDo85aPN4gSxJ7oEIOlKd7aGENZFXAM1kbIn1Ds+61gh/K1CQWSyepgJo3nUpAwH6D1ezDWV7Iaf8ueoITc8Uw==", + "integrity": "sha1-W+dEE5TQ6ldY8o25IqUAT/PwO+w=", "requires": { "@google-cloud/common": "^0.32.1", "builtin-modules": "^3.0.0", @@ -188,12 +188,12 @@ "@protobufjs/base64": { "version": "1.1.2", "resolved": "https://registry.npmjs.org/@protobufjs/base64/-/base64-1.1.2.tgz", - "integrity": "sha512-AZkcAA5vnN/v4PDqKyMR5lx7hZttPDgClv83E//FMNhR2TMcLUhfRUBHCmSl0oi9zMgDDqRUJkSxO3wm85+XLg==" + "integrity": "sha1-TIVzDlm5ofHzSQR9vyQpYDS7JzU=" }, "@protobufjs/codegen": { "version": "2.0.4", "resolved": "https://registry.npmjs.org/@protobufjs/codegen/-/codegen-2.0.4.tgz", - "integrity": "sha512-YyFaikqM5sH0ziFZCN3xDC7zeGaB/d0IUb9CATugHWbd1FRFwWwt4ld4OYMPWu5a3Xe01mGAULCdqhMlPl29Jg==" + "integrity": "sha1-fvN/DQEPsCitGtWXIuUG2SYoFcs=" }, "@protobufjs/eventemitter": { "version": "1.1.0", @@ -237,22 +237,22 @@ "@sindresorhus/is": { "version": "0.15.0", "resolved": "https://registry.npmjs.org/@sindresorhus/is/-/is-0.15.0.tgz", - "integrity": "sha512-lu8BpxjAtRCAo5ifytTpCPCj99LF7o/2Myn+NXyNCBqvPYn7Pjd76AMmUB5l7XF1U6t0hcWrlEM5ESufW7wAeA==" + "integrity": "sha1-lpFbqgXmpqHRN7rfSYTT/AWCC7Y=" }, "@types/caseless": { "version": "0.12.2", "resolved": "https://registry.npmjs.org/@types/caseless/-/caseless-0.12.2.tgz", - "integrity": "sha512-6ckxMjBBD8URvjB6J3NcnuAn5Pkl7t3TizAg+xdlzzQGSPSmBcXf8KoIH0ua/i+tio+ZRUHEXp0HEmvaR4kt0w==" + "integrity": "sha1-9l09Y4ngHutFi9VNyPUrlalGO8g=" }, "@types/console-log-level": { "version": "1.4.0", "resolved": "https://registry.npmjs.org/@types/console-log-level/-/console-log-level-1.4.0.tgz", - "integrity": "sha512-x+OscEQwcx5Biair4enH7ov9W+clcqUWaZRaxn5IkT4yNWWjRr2oiYDkY/x1uXSTVZOQ2xlbFQySaQGB+VdXGQ==" + "integrity": "sha1-7/ccQa689RyLpa2LBdfVQkviuPM=" }, "@types/duplexify": { "version": "3.6.0", "resolved": "https://registry.npmjs.org/@types/duplexify/-/duplexify-3.6.0.tgz", - "integrity": "sha512-5zOA53RUlzN74bvrSGwjudssD9F3a797sDZQkiYpUOxW+WHaXTCPz4/d5Dgi6FKnOqZ2CpaTo0DhgIfsXAOE/A==", + "integrity": "sha1-38grZL06IWj1vSZESvFlvwI33Ng=", "requires": { "@types/node": "*" } @@ -293,7 +293,7 @@ "@types/semver": { "version": "5.5.0", "resolved": "https://registry.npmjs.org/@types/semver/-/semver-5.5.0.tgz", - "integrity": "sha512-41qEJgBH/TWgo5NFSvBCJ1qkoi3Q6ONSF2avrHq1LVEZfYpdHmj0y9SuTK+u9ZhG1sYQKBL1AWXKyLWP4RaUoQ==" + "integrity": "sha1-FGwqKe59O65L8vyydGNuJkyBPEU=" }, "@types/tough-cookie": { "version": "2.3.6", @@ -303,18 +303,33 @@ "abort-controller": { "version": "3.0.0", "resolved": "https://registry.npmjs.org/abort-controller/-/abort-controller-3.0.0.tgz", - "integrity": "sha512-h8lQ8tacZYnR3vNQTgibj+tODHI5/+l06Au2Pcriv/Gmet0eaj4TwWH41sO9wnHDiQsEj19q0drzdWdeAHtweg==", + "integrity": "sha1-6vVNU7YrrkE46AnKIlyEOabvs5I=", "requires": { "event-target-shim": "^5.0.0" } }, "accepts": { - "version": "1.3.5", - "resolved": "https://registry.npmjs.org/accepts/-/accepts-1.3.5.tgz", - "integrity": "sha1-63d99gEXI6OxTopywIBcjoZ0a9I=", + "version": "1.3.7", + "resolved": "https://registry.npmjs.org/accepts/-/accepts-1.3.7.tgz", + "integrity": "sha512-Il80Qs2WjYlJIBNzNkK6KYqlVMTbZLXgHx2oT0pU/fjRHyEp+PEfEPY0R3WCwAGVOtauxh1hOxNgIf5bv7dQpA==", "requires": { - "mime-types": "~2.1.18", - "negotiator": "0.6.1" + "mime-types": "~2.1.24", + "negotiator": "0.6.2" + }, + "dependencies": { + "mime-db": { + "version": "1.44.0", + "resolved": "https://registry.npmjs.org/mime-db/-/mime-db-1.44.0.tgz", + "integrity": "sha512-/NOTfLrsPBVeH7YtFPgsVWveuL+4SjjYxaQ1xtM1KMFj7HdxlBlxeyNLzhyJVx7r4rZGJAZ/6lkKCitSc/Nmpg==" + }, + "mime-types": { + "version": "2.1.27", + "resolved": "https://registry.npmjs.org/mime-types/-/mime-types-2.1.27.tgz", + "integrity": "sha512-JIhqnCasI9yD+SsmkquHBxTSEuZdQX5BuQnS2Vc7puQQQ+8yiP5AY5uWhpdv4YL4VM5c6iliiYWPgJ/nJQLp7w==", + "requires": { + "mime-db": "1.44.0" + } + } } }, "acorn": { @@ -333,17 +348,17 @@ "agent-base": { "version": "4.3.0", "resolved": "https://registry.npmjs.org/agent-base/-/agent-base-4.3.0.tgz", - "integrity": "sha512-salcGninV0nPrwpGNn4VTXBb1SOuXQBiqbrNXoeizJsHrsL6ERFM2Ne3JUSBWRE6aeNJI2ROP/WEEIDUiDe3cg==", + "integrity": "sha1-gWXwHENgCbzK0LHRIvBe13Dvxu4=", "requires": { "es6-promisify": "^5.0.0" } }, "ajv": { - "version": "6.7.0", - "resolved": "https://registry.npmjs.org/ajv/-/ajv-6.7.0.tgz", - "integrity": "sha1-4857s3LWV3uxg58d/fy/WtKUjZY=", + "version": "6.12.2", + "resolved": "https://registry.npmjs.org/ajv/-/ajv-6.12.2.tgz", + "integrity": "sha512-k+V+hzjm5q/Mr8ef/1Y9goCmlsK4I6Sm74teeyGvFk1XrOsbsKLjEdrvny42CZ+a8sXbk8KWpY/bDwS+FLL2UQ==", "requires": { - "fast-deep-equal": "^2.0.1", + "fast-deep-equal": "^3.1.1", "fast-json-stable-stringify": "^2.0.0", "json-schema-traverse": "^0.4.1", "uri-js": "^4.2.2" @@ -357,7 +372,15 @@ "arrify": { "version": "2.0.1", "resolved": "https://registry.npmjs.org/arrify/-/arrify-2.0.1.tgz", - "integrity": "sha512-3duEwti880xqi4eAMN8AyR4a0ByT90zoYdLlevfrvU43vb0YZwZVfxOgxWrLXXXpyugL0hNZc9G6BiB5B3nUug==" + "integrity": "sha1-yWVekzHgq81YjSp8rX6ZVvZnAfo=" + }, + "asn1": { + "version": "0.2.4", + "resolved": "https://registry.npmjs.org/asn1/-/asn1-0.2.4.tgz", + "integrity": "sha512-jxwzQpLQjSmWXgwaCZE9Nz+glAG01yF1QnWgbhGwHI5A6FRIEY6IVqtHhIepHqI7/kyEyQEagBC5mBEFlIYvdg==", + "requires": { + "safer-buffer": "~2.1.0" + } }, "assert-plus": { "version": "1.0.0", @@ -378,7 +401,7 @@ "async-listener": { "version": "0.6.10", "resolved": "https://registry.npmjs.org/async-listener/-/async-listener-0.6.10.tgz", - "integrity": "sha512-gpuo6xOyF4D5DE5WvyqZdPA3NGhiT6Qf07l7DCB0wwDEsLvDIbCr6j9S5aj5Ch96dLace5tXVzWBZkxU/c5ohw==", + "integrity": "sha1-p8l6vlcLpgLXgic8DeYKUePhfLw=", "requires": { "semver": "^5.3.0", "shimmer": "^1.1.0" @@ -402,14 +425,14 @@ "integrity": "sha1-tG6JCTSpWR8tL2+G1+ap8bP+dqg=" }, "aws4": { - "version": "1.8.0", - "resolved": "https://registry.npmjs.org/aws4/-/aws4-1.8.0.tgz", - "integrity": "sha1-8OAD2cqef1nHpQiUXXsu+aBKVC8=" + "version": "1.10.0", + "resolved": "https://registry.npmjs.org/aws4/-/aws4-1.10.0.tgz", + "integrity": "sha512-3YDiu347mtVtjpyV3u5kVqQLP242c06zwDOgpeRnybmXlYYsLbtTrUBUm8i8srONt+FWobl5aibnU1030PeeuA==" }, "axios": { "version": "0.18.1", "resolved": "https://registry.npmjs.org/axios/-/axios-0.18.1.tgz", - "integrity": "sha512-0BfJq4NSfQXd+SkFdrvFbG7addhYSBA2mQwISr46pD6E5iqkWg02RAs8vyTT/j0RTnoYmeXauBuSv1qKwR179g==", + "integrity": "sha1-/z8N4ue10YDnV62YAA8Qgbh7zqM=", "requires": { "follow-redirects": "1.5.10", "is-buffer": "^2.0.2" @@ -425,12 +448,6 @@ "resolved": "https://registry.npmjs.org/base64-js/-/base64-js-1.3.1.tgz", "integrity": "sha512-mLQ4i2QO1ytvGWFWmcngKO//JXAQueZvwEKtjgQFM4jIK0kU+ytMfplL8j+n5mspOfjHwoAg+9yhb7BwAHm36g==" }, - "base64-url": { - "version": "1.2.1", - "resolved": "https://registry.npmjs.org/base64-url/-/base64-url-1.2.1.tgz", - "integrity": "sha1-GZ/WYXAqDnt9yubgaYuwicUvbXg=", - "dev": true - }, "base64id": { "version": "0.1.0", "resolved": "https://registry.npmjs.org/base64id/-/base64id-0.1.0.tgz", @@ -452,12 +469,12 @@ "bignumber.js": { "version": "7.2.1", "resolved": "https://registry.npmjs.org/bignumber.js/-/bignumber.js-7.2.1.tgz", - "integrity": "sha512-S4XzBk5sMB+Rcb/LNcpzXr57VRTxgAvaAEDAl1AwRx27j00hT84O6OkteE7u8UB3NuaaygCRrEpqox4uDOrbdQ==" + "integrity": "sha1-gMBIdZ2CaACAfEv9Uh5Q7bulel8=" }, "bindings": { "version": "1.5.0", "resolved": "https://registry.npmjs.org/bindings/-/bindings-1.5.0.tgz", - "integrity": "sha512-p2q/t/mhvuOj/UeLlV6566GD/guowlr0hHxClI0W9m7MWYkL1F0hLo+0Aexs9HSPCtR1SXQ0TD3MMKrXZajbiQ==", + "integrity": "sha1-EDU8npRTNLwFEabZCzj7x8nFBN8=", "requires": { "file-uri-to-path": "1.0.0" } @@ -468,20 +485,20 @@ "integrity": "sha1-DmVcm5wkNeqraL9AJyJtK1WjRSQ=" }, "body-parser": { - "version": "1.18.3", - "resolved": "https://registry.npmjs.org/body-parser/-/body-parser-1.18.3.tgz", - "integrity": "sha1-WykhmP/dVTs6DyDe0FkrlWlVyLQ=", + "version": "1.19.0", + "resolved": "https://registry.npmjs.org/body-parser/-/body-parser-1.19.0.tgz", + "integrity": "sha512-dhEPs72UPbDnAQJ9ZKMNTP6ptJaionhP5cBb541nXPlW60Jepo9RV/a4fX4XWW9CuFNK22krhrj1+rgzifNCsw==", "requires": { - "bytes": "3.0.0", + "bytes": "3.1.0", "content-type": "~1.0.4", "debug": "2.6.9", "depd": "~1.1.2", - "http-errors": "~1.6.3", - "iconv-lite": "0.4.23", + "http-errors": "1.7.2", + "iconv-lite": "0.4.24", "on-finished": "~2.3.0", - "qs": "6.5.2", - "raw-body": "2.3.3", - "type-is": "~1.6.16" + "qs": "6.7.0", + "raw-body": "2.4.0", + "type-is": "~1.6.17" } }, "brace-expansion": { @@ -507,7 +524,7 @@ "builtin-modules": { "version": "3.1.0", "resolved": "https://registry.npmjs.org/builtin-modules/-/builtin-modules-3.1.0.tgz", - "integrity": "sha512-k0KL0aWZuBt2lrxrcASWDfwOLMnodeQjodT/1SxEQAXsHANgo6ZC/VEaSEHCXt7aSTZ4/4H5LKa+tBXmW7Vtvw==" + "integrity": "sha1-qtl8FRMet2tltQ7yCOdYTNdqdIQ=" }, "bunyan": { "version": "0.22.3", @@ -520,9 +537,9 @@ } }, "bytes": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/bytes/-/bytes-3.0.0.tgz", - "integrity": "sha1-0ygVQE1olpn4Wk6k+odV3ROpYEg=" + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/bytes/-/bytes-3.1.0.tgz", + "integrity": "sha512-zauLjrfCG+xvoyaqLoV8bLVXXNGC4JqlxFCutSDWA6fJrTo2ZuvLYTqZ7aHBLZSMOopbzwv8f+wZcVzfVTI2Dg==" }, "caseless": { "version": "0.12.0", @@ -584,22 +601,32 @@ "console-log-level": { "version": "1.4.1", "resolved": "https://registry.npmjs.org/console-log-level/-/console-log-level-1.4.1.tgz", - "integrity": "sha512-VZzbIORbP+PPcN/gg3DXClTLPLg5Slwd5fL2MIc+o1qZ4BXBvWyc6QxPk6T/Mkr6IVjRpoAGf32XxP3ZWMVRcQ==" + "integrity": "sha1-nFprue8e9lsFq6gwKLD/iUzfYwo=" }, "content-disposition": { - "version": "0.5.2", - "resolved": "https://registry.npmjs.org/content-disposition/-/content-disposition-0.5.2.tgz", - "integrity": "sha1-DPaLud318r55YcOoUXjLhdunjLQ=" + "version": "0.5.3", + "resolved": "https://registry.npmjs.org/content-disposition/-/content-disposition-0.5.3.tgz", + "integrity": "sha512-ExO0774ikEObIAEV9kDo50o+79VCUdEB6n6lzKgGwupcVeRlhrj3qGAfwq8G6uBJjkqLrhT0qEYFcWng8z1z0g==", + "requires": { + "safe-buffer": "5.1.2" + }, + "dependencies": { + "safe-buffer": { + "version": "5.1.2", + "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.1.2.tgz", + "integrity": "sha512-Gd2UZBJDkXlY7GbJxfsE8/nvKkUEU1G38c1siN6QP6a9PT9MmHB8GnpscSmMJSoF8LOIrt8ud/wPtojys4G6+g==" + } + } }, "content-type": { "version": "1.0.4", "resolved": "https://registry.npmjs.org/content-type/-/content-type-1.0.4.tgz", - "integrity": "sha1-4TjMdeBAxyexlm/l5fjJruJW/js=" + "integrity": "sha512-hIP3EEPs8tB9AT1L+NUqtwOAps4mk2Zob89MWXMHjHWg9milF/j4osnnQLXBCBFBk/tvIG/tUc9mOUJiPBhPXA==" }, "continuation-local-storage": { "version": "3.2.1", "resolved": "https://registry.npmjs.org/continuation-local-storage/-/continuation-local-storage-3.2.1.tgz", - "integrity": "sha512-jx44cconVqkCEEyLSKWwkvUXwO561jXMa3LPjTPsm5QR22PA0/mhe33FT4Xb5y74JDvt/Cq+5lm8S8rskLv9ZA==", + "integrity": "sha1-EfYT906RT+mzTJKtLSj+auHbf/s=", "requires": { "async-listener": "^0.6.0", "emitter-listener": "^1.1.1" @@ -611,48 +638,49 @@ "integrity": "sha1-5+Ch+e9DtMi6klxcWpboBtFoc7s=" }, "cookie-parser": { - "version": "1.4.3", - "resolved": "https://registry.npmjs.org/cookie-parser/-/cookie-parser-1.4.3.tgz", - "integrity": "sha1-D+MfoZ0AC5X0qt8fU/3CuKIDuqU=", + "version": "1.4.5", + "resolved": "https://registry.npmjs.org/cookie-parser/-/cookie-parser-1.4.5.tgz", + "integrity": "sha512-f13bPUj/gG/5mDr+xLmSxxDsB9DQiTIfhJS/sqjrmfAWiAN+x2O4i/XguTL9yDZ+/IFDanJ+5x7hC4CXT9Tdzw==", "requires": { - "cookie": "0.3.1", + "cookie": "0.4.0", "cookie-signature": "1.0.6" + }, + "dependencies": { + "cookie": { + "version": "0.4.0", + "resolved": "https://registry.npmjs.org/cookie/-/cookie-0.4.0.tgz", + "integrity": "sha512-+Hp8fLp57wnUSt0tY0tHEXh4voZRDnoIrZPqlo3DPiI4y9lwg/jqx+1Om94/W6ZaPDOUbnjOt/99w66zk+l1Xg==" + }, + "cookie-signature": { + "version": "1.0.6", + "resolved": "https://registry.npmjs.org/cookie-signature/-/cookie-signature-1.0.6.tgz", + "integrity": "sha1-4wOogrNCzD7oylE6eZmXNNqzriw=" + } } }, "cookie-signature": { - "version": "1.0.6", - "resolved": "https://registry.npmjs.org/cookie-signature/-/cookie-signature-1.0.6.tgz", - "integrity": "sha1-4wOogrNCzD7oylE6eZmXNNqzriw=" + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/cookie-signature/-/cookie-signature-1.1.0.tgz", + "integrity": "sha512-Alvs19Vgq07eunykd3Xy2jF0/qSNv2u7KDbAek9H5liV1UMijbqFs5cycZvv5dVsvseT/U4H8/7/w8Koh35C4A==", + "dev": true }, "core-util-is": { "version": "1.0.2", "resolved": "https://registry.npmjs.org/core-util-is/-/core-util-is-1.0.2.tgz", "integrity": "sha1-tf1UIgqivFq1eqtxQMlAdUUDwac=" }, - "crc": { - "version": "3.4.4", - "resolved": "https://registry.npmjs.org/crc/-/crc-3.4.4.tgz", - "integrity": "sha1-naHpgOO9RPxck79as9ozeNheRms=" - }, "dashdash": { "version": "1.14.1", "resolved": "https://registry.npmjs.org/dashdash/-/dashdash-1.14.1.tgz", "integrity": "sha1-hTz6D3y+L+1d4gMmuN1YEDX24vA=", "requires": { "assert-plus": "^1.0.0" - }, - "dependencies": { - "assert-plus": { - "version": "1.0.0", - "resolved": "https://registry.npmjs.org/assert-plus/-/assert-plus-1.0.0.tgz", - "integrity": "sha1-8S4PPF13sLHN2RRpQuTpbB5N1SU=" - } } }, "debug": { "version": "2.6.9", "resolved": "https://registry.npmjs.org/debug/-/debug-2.6.9.tgz", - "integrity": "sha1-XRKFFd8TT/Mn6QpMk/Tgd6U2NB8=", + "integrity": "sha512-bC7ElrdJaJnPbAP+1EotYvqZsb3ecl5wi6Bfi6BJTUcNowp6cvspg0jXznRTKDjm/E7AdgFBVeAPVMNcKGsHMA==", "requires": { "ms": "2.0.0" } @@ -669,7 +697,7 @@ "delay": { "version": "4.3.0", "resolved": "https://registry.npmjs.org/delay/-/delay-4.3.0.tgz", - "integrity": "sha512-Lwaf3zVFDMBop1yDuFZ19F9WyGcZcGacsbdlZtWjQmM50tOcMntm1njF/Nb/Vjij3KaSvCF+sEYGKrrjObu2NA==" + "integrity": "sha1-7+6/uPVFV5yzlrOnIkQ+yW0UxQ4=" }, "delayed-stream": { "version": "1.0.0", @@ -707,7 +735,7 @@ "duplexify": { "version": "3.7.1", "resolved": "https://registry.npmjs.org/duplexify/-/duplexify-3.7.1.tgz", - "integrity": "sha512-07z8uv2wMyS51kKhD1KsdXJg5WQ6t93RneqRxUHnskXVtlYYkLqM0gqStQZ3pj073g687jPCHrqNfCzawLYh5g==", + "integrity": "sha1-Kk31MX9sz9kfhtb9JdjYoQO4gwk=", "requires": { "end-of-stream": "^1.0.0", "inherits": "^2.0.1", @@ -727,7 +755,7 @@ "ecdsa-sig-formatter": { "version": "1.0.11", "resolved": "https://registry.npmjs.org/ecdsa-sig-formatter/-/ecdsa-sig-formatter-1.0.11.tgz", - "integrity": "sha512-nagl3RYrbNv6kQkeJIpt6NJZy8twLB/2vtz6yN9Z4vRKHN4/QZJIEbqohALSgwKdnksuY3k5Addp5lg8sVoVcQ==", + "integrity": "sha1-rg8PothQRe8UqBfao86azQSJ5b8=", "requires": { "safe-buffer": "^5.0.1" } @@ -740,7 +768,7 @@ "emitter-listener": { "version": "1.1.2", "resolved": "https://registry.npmjs.org/emitter-listener/-/emitter-listener-1.1.2.tgz", - "integrity": "sha512-Bt1sBAGFHY9DKY+4/2cV6izcKJUf5T7/gkdmkxzX/qv9CcGH8xSwVRW5mtX03SWJtRTWSOpzCuWN9rBFYZepZQ==", + "integrity": "sha1-VrFA6PaZI3Wz18ssqxzHQy2WMug=", "requires": { "shimmer": "^1.2.0" } @@ -766,7 +794,7 @@ "es6-promise": { "version": "4.2.8", "resolved": "https://registry.npmjs.org/es6-promise/-/es6-promise-4.2.8.tgz", - "integrity": "sha512-HJDGx5daxeIvxdBxvG2cb9g4tEvwIk3i8+nhX0yGrYmZUzbkdg8QbDevheDB8gd0//uPj4c1EQua8Q+MViT0/w==" + "integrity": "sha1-TrIVlMlyvEBVPSduUQU5FD21Pgo=" }, "es6-promisify": { "version": "5.0.0", @@ -795,136 +823,96 @@ "event-target-shim": { "version": "5.0.1", "resolved": "https://registry.npmjs.org/event-target-shim/-/event-target-shim-5.0.1.tgz", - "integrity": "sha512-i/2XbnSz/uxRCU6+NdVJgKWDTM427+MqYbkQzD321DuCQJUqOuJKIA0IM2+W2xtYHdKOmZ4dR6fExsd4SXL+WQ==" + "integrity": "sha1-XU0+vflYPWOlMzzi3rdICrKwV4k=" }, "express": { - "version": "4.16.3", - "resolved": "https://registry.npmjs.org/express/-/express-4.16.3.tgz", - "integrity": "sha1-avilAjUNsyRuzEvs9rWjTSL37VM=", + "version": "4.17.1", + "resolved": "https://registry.npmjs.org/express/-/express-4.17.1.tgz", + "integrity": "sha512-mHJ9O79RqluphRrcw2X/GTh3k9tVv8YcoyY4Kkh4WDMUYKRZUq0h1o0w2rrrxBqM7VoeUVqgb27xlEMXTnYt4g==", "requires": { - "accepts": "~1.3.5", + "accepts": "~1.3.7", "array-flatten": "1.1.1", - "body-parser": "1.18.2", - "content-disposition": "0.5.2", + "body-parser": "1.19.0", + "content-disposition": "0.5.3", "content-type": "~1.0.4", - "cookie": "0.3.1", + "cookie": "0.4.0", "cookie-signature": "1.0.6", "debug": "2.6.9", "depd": "~1.1.2", "encodeurl": "~1.0.2", "escape-html": "~1.0.3", "etag": "~1.8.1", - "finalhandler": "1.1.1", + "finalhandler": "~1.1.2", "fresh": "0.5.2", "merge-descriptors": "1.0.1", "methods": "~1.1.2", "on-finished": "~2.3.0", - "parseurl": "~1.3.2", + "parseurl": "~1.3.3", "path-to-regexp": "0.1.7", - "proxy-addr": "~2.0.3", - "qs": "6.5.1", - "range-parser": "~1.2.0", - "safe-buffer": "5.1.1", - "send": "0.16.2", - "serve-static": "1.13.2", - "setprototypeof": "1.1.0", - "statuses": "~1.4.0", - "type-is": "~1.6.16", + "proxy-addr": "~2.0.5", + "qs": "6.7.0", + "range-parser": "~1.2.1", + "safe-buffer": "5.1.2", + "send": "0.17.1", + "serve-static": "1.14.1", + "setprototypeof": "1.1.1", + "statuses": "~1.5.0", + "type-is": "~1.6.18", "utils-merge": "1.0.1", "vary": "~1.1.2" }, "dependencies": { - "body-parser": { - "version": "1.18.2", - "resolved": "https://registry.npmjs.org/body-parser/-/body-parser-1.18.2.tgz", - "integrity": "sha1-h2eKGdhLR9hZuDGZvVm84iKxBFQ=", - "requires": { - "bytes": "3.0.0", - "content-type": "~1.0.4", - "debug": "2.6.9", - "depd": "~1.1.1", - "http-errors": "~1.6.2", - "iconv-lite": "0.4.19", - "on-finished": "~2.3.0", - "qs": "6.5.1", - "raw-body": "2.3.2", - "type-is": "~1.6.15" - } + "cookie": { + "version": "0.4.0", + "resolved": "https://registry.npmjs.org/cookie/-/cookie-0.4.0.tgz", + "integrity": "sha512-+Hp8fLp57wnUSt0tY0tHEXh4voZRDnoIrZPqlo3DPiI4y9lwg/jqx+1Om94/W6ZaPDOUbnjOt/99w66zk+l1Xg==" }, - "iconv-lite": { - "version": "0.4.19", - "resolved": "https://registry.npmjs.org/iconv-lite/-/iconv-lite-0.4.19.tgz", - "integrity": "sha1-90aPYBNfXl2tM5nAqBvpoWA6CCs=" + "cookie-signature": { + "version": "1.0.6", + "resolved": "https://registry.npmjs.org/cookie-signature/-/cookie-signature-1.0.6.tgz", + "integrity": "sha1-4wOogrNCzD7oylE6eZmXNNqzriw=" }, - "qs": { - "version": "6.5.1", - "resolved": "https://registry.npmjs.org/qs/-/qs-6.5.1.tgz", - "integrity": "sha1-NJzfbu+J7EXBLX1es/wMhwNDptg=" - }, - "raw-body": { - "version": "2.3.2", - "resolved": "https://registry.npmjs.org/raw-body/-/raw-body-2.3.2.tgz", - "integrity": "sha1-vNYMd9Prk83gBQKVw/N5OJvIj4k=", - "requires": { - "bytes": "3.0.0", - "http-errors": "1.6.2", - "iconv-lite": "0.4.19", - "unpipe": "1.0.0" - }, - "dependencies": { - "depd": { - "version": "1.1.1", - "resolved": "https://registry.npmjs.org/depd/-/depd-1.1.1.tgz", - "integrity": "sha1-V4O04cRZ8G+lyif5kfPQbnoxA1k=" - }, - "http-errors": { - "version": "1.6.2", - "resolved": "https://registry.npmjs.org/http-errors/-/http-errors-1.6.2.tgz", - "integrity": "sha1-CgAsyFcHGSp+eUbO7cERVfYOxzY=", - "requires": { - "depd": "1.1.1", - "inherits": "2.0.3", - "setprototypeof": "1.0.3", - "statuses": ">= 1.3.1 < 2" - } - }, - "setprototypeof": { - "version": "1.0.3", - "resolved": "https://registry.npmjs.org/setprototypeof/-/setprototypeof-1.0.3.tgz", - "integrity": "sha1-ZlZ+NwQ+608E2RvWWMDL77VbjgQ=" - } - } - }, - "statuses": { - "version": "1.4.0", - "resolved": "https://registry.npmjs.org/statuses/-/statuses-1.4.0.tgz", - "integrity": "sha1-u3PURtonlhBu/MG2AaJT1sRr0Ic=" + "safe-buffer": { + "version": "5.1.2", + "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.1.2.tgz", + "integrity": "sha512-Gd2UZBJDkXlY7GbJxfsE8/nvKkUEU1G38c1siN6QP6a9PT9MmHB8GnpscSmMJSoF8LOIrt8ud/wPtojys4G6+g==" } } }, "express-session": { - "version": "1.15.6", - "resolved": "https://registry.npmjs.org/express-session/-/express-session-1.15.6.tgz", - "integrity": "sha1-R7QWDIj0KrcP6KUI4xy/92dXqwo=", + "version": "1.17.1", + "resolved": "https://registry.npmjs.org/express-session/-/express-session-1.17.1.tgz", + "integrity": "sha512-UbHwgqjxQZJiWRTMyhvWGvjBQduGCSBDhhZXYenziMFjxst5rMV+aJZ6hKPHZnPyHGsrqRICxtX8jtEbm/z36Q==", "requires": { - "cookie": "0.3.1", + "cookie": "0.4.0", "cookie-signature": "1.0.6", - "crc": "3.4.4", "debug": "2.6.9", - "depd": "~1.1.1", - "on-headers": "~1.0.1", - "parseurl": "~1.3.2", - "uid-safe": "~2.1.5", - "utils-merge": "1.0.1" + "depd": "~2.0.0", + "on-headers": "~1.0.2", + "parseurl": "~1.3.3", + "safe-buffer": "5.2.0", + "uid-safe": "~2.1.5" }, "dependencies": { - "uid-safe": { - "version": "2.1.5", - "resolved": "https://registry.npmjs.org/uid-safe/-/uid-safe-2.1.5.tgz", - "integrity": "sha1-Kz1cckDo/C5Y+Komnl7knAhXvTo=", - "requires": { - "random-bytes": "~1.0.0" - } + "cookie": { + "version": "0.4.0", + "resolved": "https://registry.npmjs.org/cookie/-/cookie-0.4.0.tgz", + "integrity": "sha512-+Hp8fLp57wnUSt0tY0tHEXh4voZRDnoIrZPqlo3DPiI4y9lwg/jqx+1Om94/W6ZaPDOUbnjOt/99w66zk+l1Xg==" + }, + "cookie-signature": { + "version": "1.0.6", + "resolved": "https://registry.npmjs.org/cookie-signature/-/cookie-signature-1.0.6.tgz", + "integrity": "sha1-4wOogrNCzD7oylE6eZmXNNqzriw=" + }, + "depd": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/depd/-/depd-2.0.0.tgz", + "integrity": "sha512-g7nH6P6dyDioJogAAGprGpCtVImJhpPk/roCzdb3fIh61/s/nPsfR6onyMwkCAR/OlC3yBC0lESvUoQEAssIrw==" + }, + "safe-buffer": { + "version": "5.2.0", + "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.2.0.tgz", + "integrity": "sha512-fZEwUGbVl7kouZs1jCdMLdt95hdIv0ZeHg6L7qPeciMZhZ+/gdesW4wgTARkrFWEpspjEATAzUGPG8N2jJiwbg==" } } }, @@ -939,14 +927,14 @@ "integrity": "sha1-lpGEQOMEGnpBT4xS48V06zw+HgU=" }, "fast-deep-equal": { - "version": "2.0.1", - "resolved": "https://registry.npmjs.org/fast-deep-equal/-/fast-deep-equal-2.0.1.tgz", - "integrity": "sha1-ewUhjd+WZ79/Nwv3/bLLFf3Qqkk=" + "version": "3.1.3", + "resolved": "https://registry.npmjs.org/fast-deep-equal/-/fast-deep-equal-3.1.3.tgz", + "integrity": "sha512-f3qQ9oQy9j2AhBe/H9VC91wLmKBCCU/gDOnKNAYG5hswO7BLKj09Hc5HYNz9cGI++xlpDCIgDaitVs03ATR84Q==" }, "fast-json-stable-stringify": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/fast-json-stable-stringify/-/fast-json-stable-stringify-2.0.0.tgz", - "integrity": "sha1-1RQsDK7msRifh9OnYREGT4bIu/I=" + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/fast-json-stable-stringify/-/fast-json-stable-stringify-2.1.0.tgz", + "integrity": "sha512-lhd/wF+Lk98HZoTCtlVraHtfh5XYijIjalXck7saUtuanSDyLMxnHhSXEDJqHxD7msR8D0uCmqlkwjCV8xvwHw==" }, "fast-text-encoding": { "version": "1.0.1", @@ -956,27 +944,20 @@ "file-uri-to-path": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/file-uri-to-path/-/file-uri-to-path-1.0.0.tgz", - "integrity": "sha512-0Zt+s3L7Vf1biwWZ29aARiVYLx7iMGnEUl9x33fbB/j3jR81u/O2LbqK+Bm1CDSNDKVtJ/YjwY7TUd5SkeLQLw==" + "integrity": "sha1-VTp7hEb/b2hDWcRF8eN6BdrMM90=" }, "finalhandler": { - "version": "1.1.1", - "resolved": "https://registry.npmjs.org/finalhandler/-/finalhandler-1.1.1.tgz", - "integrity": "sha1-7r9O2EAHnIP0JJA4ydcDAIMBsQU=", + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/finalhandler/-/finalhandler-1.1.2.tgz", + "integrity": "sha512-aAWcW57uxVNrQZqFXjITpW3sIUQmHGG3qSb9mUah9MgMC4NeWhNOlNjXEYq3HjRAvL6arUviZGGJsBg6z0zsWA==", "requires": { "debug": "2.6.9", "encodeurl": "~1.0.2", "escape-html": "~1.0.3", "on-finished": "~2.3.0", - "parseurl": "~1.3.2", - "statuses": "~1.4.0", + "parseurl": "~1.3.3", + "statuses": "~1.5.0", "unpipe": "~1.0.0" - }, - "dependencies": { - "statuses": { - "version": "1.4.0", - "resolved": "https://registry.npmjs.org/statuses/-/statuses-1.4.0.tgz", - "integrity": "sha1-u3PURtonlhBu/MG2AaJT1sRr0Ic=" - } } }, "findit2": { @@ -987,7 +968,7 @@ "follow-redirects": { "version": "1.5.10", "resolved": "https://registry.npmjs.org/follow-redirects/-/follow-redirects-1.5.10.tgz", - "integrity": "sha512-0V5l4Cizzvqt5D44aTXbFZz+FtyXV1vrDN6qrelxtfYQKW0KO0W2T/hkE8xvGa/540LkZlkaUjO4ailYTFtHVQ==", + "integrity": "sha1-e3qfmuov3/NnhqlP9kPtB/T/Xio=", "requires": { "debug": "=3.1.0" }, @@ -995,7 +976,7 @@ "debug": { "version": "3.1.0", "resolved": "https://registry.npmjs.org/debug/-/debug-3.1.0.tgz", - "integrity": "sha512-OX8XqP7/1a9cqkxYw2yXss15f26NKWBpDXQd0/uK/KPqdQhxbPa994hnzjcE2VqQpDslf55723cKPUOGSmMY3g==", + "integrity": "sha1-W7WgZyYotkFJVmuhaBnmFRjGcmE=", "requires": { "ms": "2.0.0" } @@ -1010,7 +991,7 @@ "form-data": { "version": "2.3.3", "resolved": "https://registry.npmjs.org/form-data/-/form-data-2.3.3.tgz", - "integrity": "sha1-3M5SwF9kTymManq5Nr1yTO/786Y=", + "integrity": "sha512-1lLKB2Mu3aGP1Q/2eCOx0fNbRMe7XdwktwOruhfqqd0rIJWwN4Dh+E3hrPSlDCXnSR7UtZ1N38rVXm+6+MEhJQ==", "requires": { "asynckit": "^0.4.0", "combined-stream": "^1.0.6", @@ -1045,7 +1026,7 @@ "gaxios": { "version": "1.8.4", "resolved": "https://registry.npmjs.org/gaxios/-/gaxios-1.8.4.tgz", - "integrity": "sha512-BoENMnu1Gav18HcpV9IleMPZ9exM+AvUjrAOV4Mzs/vfz2Lu/ABv451iEXByKiMPn2M140uul1txXCg83sAENw==", + "integrity": "sha1-4Iw0/pPAqbZ6Ure556ZOZDX5ozk=", "requires": { "abort-controller": "^3.0.0", "extend": "^3.0.2", @@ -1056,7 +1037,7 @@ "gcp-metadata": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/gcp-metadata/-/gcp-metadata-1.0.0.tgz", - "integrity": "sha512-Q6HrgfrCQeEircnNP3rCcEgiDv7eF9+1B+1MMgpE190+/+0mjQR8PxeOaRgxZWmdDAF9EIryHB9g1moPiw1SbQ==", + "integrity": "sha1-UhJEAin6CZ/C98KlzcuVV16bLKY=", "requires": { "gaxios": "^1.0.2", "json-bigint": "^0.3.0" @@ -1068,13 +1049,6 @@ "integrity": "sha1-Xv+OPmhNVprkyysSgmBOi6YhSfo=", "requires": { "assert-plus": "^1.0.0" - }, - "dependencies": { - "assert-plus": { - "version": "1.0.0", - "resolved": "https://registry.npmjs.org/assert-plus/-/assert-plus-1.0.0.tgz", - "integrity": "sha1-8S4PPF13sLHN2RRpQuTpbB5N1SU=" - } } }, "glob": { @@ -1093,7 +1067,7 @@ "google-auth-library": { "version": "3.1.2", "resolved": "https://registry.npmjs.org/google-auth-library/-/google-auth-library-3.1.2.tgz", - "integrity": "sha512-cDQMzTotwyWMrg5jRO7q0A4TL/3GWBgO7I7q5xGKNiiFf9SmGY/OJ1YsLMgI2MVHHsEGyrqYnbnmV1AE+Z6DnQ==", + "integrity": "sha1-/y+IzVzSEYpXvT1a08CTyIN/w1A=", "requires": { "base64-js": "^1.3.0", "fast-text-encoding": "^1.0.0", @@ -1116,7 +1090,7 @@ "google-p12-pem": { "version": "1.0.4", "resolved": "https://registry.npmjs.org/google-p12-pem/-/google-p12-pem-1.0.4.tgz", - "integrity": "sha512-SwLAUJqUfTB2iS+wFfSS/G9p7bt4eWcc2LyfvmUXe7cWp6p3mpxDo6LLI29MXdU6wvPcQ/up298X7GMC5ylAlA==", + "integrity": "sha1-t3+4M6Lrn388aJ4uVPCVJ293dgU=", "requires": { "node-forge": "^0.8.0", "pify": "^4.0.0" @@ -1125,7 +1099,7 @@ "gtoken": { "version": "2.3.3", "resolved": "https://registry.npmjs.org/gtoken/-/gtoken-2.3.3.tgz", - "integrity": "sha512-EaB49bu/TCoNeQjhCYKI/CurooBKkGxIqFHsWABW0b25fobBYVTMe84A8EBVVZhl8emiUdNypil9huMOTmyAnw==", + "integrity": "sha1-in/hVcXODEtxyIbPsoKpBg2UpkE=", "requires": { "gaxios": "^1.0.4", "google-p12-pem": "^1.0.0", @@ -1137,7 +1111,7 @@ "mime": { "version": "2.4.4", "resolved": "https://registry.npmjs.org/mime/-/mime-2.4.4.tgz", - "integrity": "sha512-LRxmNwziLPT828z+4YkNzloCFC2YM4wrB99k+AV5ZbEyfGNWfG8SO1FUXLmLDBSo89NrJZ4DIWeLjy1CHGhMGA==" + "integrity": "sha1-vXuRE1/GsBzePpuuM9ZZtj2IV+U=" } } }, @@ -1149,7 +1123,7 @@ "har-validator": { "version": "5.1.3", "resolved": "https://registry.npmjs.org/har-validator/-/har-validator-5.1.3.tgz", - "integrity": "sha1-HvievT5JllV2de7ZiTEQ3DUPoIA=", + "integrity": "sha512-sNvOCzEQNr/qrvJgc3UG/kD4QtlHycrzwS+6mfTrrSq97BvaYcPZZI1ZSqGSPR73Cxn4LKTD4PttRwfU7jWq5g==", "requires": { "ajv": "^6.5.5", "har-schema": "^2.0.0" @@ -1164,17 +1138,18 @@ "hex2dec": { "version": "1.1.2", "resolved": "https://registry.npmjs.org/hex2dec/-/hex2dec-1.1.2.tgz", - "integrity": "sha512-Yu+q/XWr2fFQ11tHxPq4p4EiNkb2y+lAacJNhAdRXVfRIcDH6gi7htWFnnlIzvqHMHoWeIsfXlNAjZInpAOJDA==" + "integrity": "sha1-jhzkvvNqdPfVcjw/swkMKGAHczg=" }, "http-errors": { - "version": "1.6.3", - "resolved": "https://registry.npmjs.org/http-errors/-/http-errors-1.6.3.tgz", - "integrity": "sha1-i1VoC7S+KDoLW/TqLjhYC+HZMg0=", + "version": "1.7.2", + "resolved": "https://registry.npmjs.org/http-errors/-/http-errors-1.7.2.tgz", + "integrity": "sha512-uUQBt3H/cSIVfch6i1EuPNy/YsRSOUBXTVfZ+yR7Zjez3qjBz6i9+i4zjNaoqcoFVI4lQJ5plg63TvGfRSDCRg==", "requires": { "depd": "~1.1.2", "inherits": "2.0.3", - "setprototypeof": "1.1.0", - "statuses": ">= 1.4.0 < 2" + "setprototypeof": "1.1.1", + "statuses": ">= 1.5.0 < 2", + "toidentifier": "1.0.0" } }, "http-signature": { @@ -1212,9 +1187,9 @@ } }, "iconv-lite": { - "version": "0.4.23", - "resolved": "https://registry.npmjs.org/iconv-lite/-/iconv-lite-0.4.23.tgz", - "integrity": "sha1-KXhx9jvlB63Pv8pxXQzQ7thOmmM=", + "version": "0.4.24", + "resolved": "https://registry.npmjs.org/iconv-lite/-/iconv-lite-0.4.24.tgz", + "integrity": "sha512-v3MXnZAcvnywkTUEZomIActle7RXXeedOR31wwl7VlyoXO4Qi9arvSenNQWne1TcRwhCL1HwLI21bEqdpj8/rA==", "requires": { "safer-buffer": ">= 2.1.2 < 3" } @@ -1234,9 +1209,9 @@ "integrity": "sha1-Yzwsg+PaQqUC9SRmAiSA9CCCYd4=" }, "ioredis": { - "version": "4.16.1", - "resolved": "https://registry.npmjs.org/ioredis/-/ioredis-4.16.1.tgz", - "integrity": "sha512-g76Mm9dE7BLuewncu1MimGZw5gDDjDwjoRony/VoSxSJEKAhuYncDEwYKYjtHi2NWsTNIB6XXRjE64uVa/wpKQ==", + "version": "4.17.3", + "resolved": "https://registry.npmjs.org/ioredis/-/ioredis-4.17.3.tgz", + "integrity": "sha512-iRvq4BOYzNFkDnSyhx7cmJNOi1x/HWYe+A4VXHBu4qpwJaGT1Mp+D2bVGJntH9K/Z/GeOM/Nprb8gB3bmitz1Q==", "requires": { "cluster-key-slot": "^1.1.0", "debug": "^4.1.1", @@ -1265,14 +1240,14 @@ } }, "ipaddr.js": { - "version": "1.6.0", - "resolved": "https://registry.npmjs.org/ipaddr.js/-/ipaddr.js-1.6.0.tgz", - "integrity": "sha1-4/o1e3c9phnybpXwSdBVxyeW+Gs=" + "version": "1.9.1", + "resolved": "https://registry.npmjs.org/ipaddr.js/-/ipaddr.js-1.9.1.tgz", + "integrity": "sha512-0KI/607xoxSToH7GjN1FfSbLoU0+btTicjsQSWQlh/hZykN8KpmMf7uYwPW3R+akZ6R/w18ZlXSHBYXiYUPO3g==" }, "is": { "version": "3.3.0", "resolved": "https://registry.npmjs.org/is/-/is-3.3.0.tgz", - "integrity": "sha512-nW24QBoPcFGGHJGUwnfpI7Yc5CdqWNdsyHQszVE/z2pKHXzh7FZ5GWhJqSyaQ9wMkQnsTx+kAI8bHlCX4tKdbg==" + "integrity": "sha1-Yc/23TxBk9uUo9YlggcrROVkXXk=" }, "is-buffer": { "version": "2.0.4", @@ -1315,7 +1290,7 @@ "json-schema-traverse": { "version": "0.4.1", "resolved": "https://registry.npmjs.org/json-schema-traverse/-/json-schema-traverse-0.4.1.tgz", - "integrity": "sha1-afaofZUTq4u4/mO9sJecRI5oRmA=" + "integrity": "sha512-xbbCH5dCYU5T8LcEhhuh7HJ88HXuW3qsI3Y0zOZFKfZEHcpWiHU/Jxzk629Brsab/mMiHQti9wMP+845RPe3Vg==" }, "json-stringify-safe": { "version": "5.0.1", @@ -1331,19 +1306,12 @@ "extsprintf": "1.3.0", "json-schema": "0.2.3", "verror": "1.10.0" - }, - "dependencies": { - "assert-plus": { - "version": "1.0.0", - "resolved": "https://registry.npmjs.org/assert-plus/-/assert-plus-1.0.0.tgz", - "integrity": "sha1-8S4PPF13sLHN2RRpQuTpbB5N1SU=" - } } }, "jwa": { "version": "1.4.1", "resolved": "https://registry.npmjs.org/jwa/-/jwa-1.4.1.tgz", - "integrity": "sha512-qiLX/xhEEFKUAJ6FiBMbes3w9ATzyk5W7Hvzpa/SLYdxNtng+gcurvrI7TbACjIXlsJyr05/S1oUhZrc63evQA==", + "integrity": "sha1-dDwymFy56YZVUw1TZBtmyGRbA5o=", "requires": { "buffer-equal-constant-time": "1.0.1", "ecdsa-sig-formatter": "1.0.11", @@ -1353,7 +1321,7 @@ "jws": { "version": "3.2.2", "resolved": "https://registry.npmjs.org/jws/-/jws-3.2.2.tgz", - "integrity": "sha512-YHlZCB6lMTllWDtSPHz/ZXTsi8S00usEV6v1tjq8tOUZzw7DpSDWVXjXDre6ed1w/pd495ODpHZYSdkRTsa0HA==", + "integrity": "sha1-ABCZ82OUaMlBQADpmZX6UvtHgwQ=", "requires": { "jwa": "^1.4.1", "safe-buffer": "^5.0.1" @@ -1408,6 +1376,61 @@ "requires": { "nan": "^2.10.0" } + }, + "mime-db": { + "version": "1.44.0", + "resolved": "https://registry.npmjs.org/mime-db/-/mime-db-1.44.0.tgz", + "integrity": "sha512-/NOTfLrsPBVeH7YtFPgsVWveuL+4SjjYxaQ1xtM1KMFj7HdxlBlxeyNLzhyJVx7r4rZGJAZ/6lkKCitSc/Nmpg==" + }, + "mime-types": { + "version": "2.1.27", + "resolved": "https://registry.npmjs.org/mime-types/-/mime-types-2.1.27.tgz", + "integrity": "sha512-JIhqnCasI9yD+SsmkquHBxTSEuZdQX5BuQnS2Vc7puQQQ+8yiP5AY5uWhpdv4YL4VM5c6iliiYWPgJ/nJQLp7w==", + "requires": { + "mime-db": "1.44.0" + } + }, + "qs": { + "version": "6.5.2", + "resolved": "https://registry.npmjs.org/qs/-/qs-6.5.2.tgz", + "integrity": "sha512-N5ZAX4/LxJmF+7wN74pUD6qAh9/wnvdQcjq9TZjevvXzSUo7bfmw91saqMjzGS2xq91/odN2dW/WOl7qQHNDGA==" + }, + "request": { + "version": "2.88.0", + "resolved": "https://registry.npmjs.org/request/-/request-2.88.0.tgz", + "integrity": "sha512-NAqBSrijGLZdM0WZNsInLJpkJokL72XYjUpnB0iwsRgxh7dB6COrHnTBNwN0E+lHDAJzu7kLAkDeY08z2/A0hg==", + "requires": { + "aws-sign2": "~0.7.0", + "aws4": "^1.8.0", + "caseless": "~0.12.0", + "combined-stream": "~1.0.6", + "extend": "~3.0.2", + "forever-agent": "~0.6.1", + "form-data": "~2.3.2", + "har-validator": "~5.1.0", + "http-signature": "~1.2.0", + "is-typedarray": "~1.0.0", + "isstream": "~0.1.2", + "json-stringify-safe": "~5.0.1", + "mime-types": "~2.1.19", + "oauth-sign": "~0.9.0", + "performance-now": "^2.1.0", + "qs": "~6.5.2", + "safe-buffer": "^5.1.2", + "tough-cookie": "~2.4.3", + "tunnel-agent": "^0.6.0", + "uuid": "^3.3.2" + } + }, + "safe-buffer": { + "version": "5.2.1", + "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.2.1.tgz", + "integrity": "sha512-rp3So07KcdmmKbGvgaNxQSJr7bGVSVk5S9Eq1F+ppbRo70+YeaDxkw5Dd8NPN+GD6bjnYm2VuPuCXmpuYvmCXQ==" + }, + "uuid": { + "version": "3.4.0", + "resolved": "https://registry.npmjs.org/uuid/-/uuid-3.4.0.tgz", + "integrity": "sha512-HjSDRw6gZE5JMggctHBcjVak08+KEVhSIiDzFnT9S9aegmp85S/bReBVTb4QTFaRNptJ9kuYaNhnbNEOkbKb/A==" } } }, @@ -1420,12 +1443,12 @@ "long": { "version": "4.0.0", "resolved": "https://registry.npmjs.org/long/-/long-4.0.0.tgz", - "integrity": "sha512-XsP+KhQif4bjX1kbuSiySJFNAehNxgLb6hPRGJ9QsUr8ajHkuXGdrHmFUTUUXhDwVX2R5bY4JNZEwbUiMhV+MA==" + "integrity": "sha1-mntxz7fTYaGU6lVSQckvdGjVvyg=" }, "lru-cache": { "version": "5.1.1", "resolved": "https://registry.npmjs.org/lru-cache/-/lru-cache-5.1.1.tgz", - "integrity": "sha512-KpNARQA3Iwv+jTA0utUVVbrh+Jlrr1Fv0e56GGzAFOXN7dk/FviaDW8LHmK52DlcH4WP2n6gI8vN1aesBFgo9w==", + "integrity": "sha1-HaJ+ZxAnGUdpXa9oSOhH8B2EuSA=", "requires": { "yallist": "^3.0.2" } @@ -1487,9 +1510,9 @@ } }, "mime": { - "version": "1.4.1", - "resolved": "https://registry.npmjs.org/mime/-/mime-1.4.1.tgz", - "integrity": "sha1-Eh+evEnjdm8xGnbh+hyAA8SwOqY=" + "version": "1.6.0", + "resolved": "https://registry.npmjs.org/mime/-/mime-1.6.0.tgz", + "integrity": "sha512-x0Vn8spI+wuJ1O6S7gnbaQg8Pxh4NNHb7KSINmEWKiPE4RKOplvijn+NkmYmmRgP68mc70j2EbeTFRsrswaQeg==" }, "mime-db": { "version": "1.33.0", @@ -1633,12 +1656,6 @@ "resolved": "https://registry.npmjs.org/nan/-/nan-2.12.1.tgz", "integrity": "sha1-exqhk+mqhgV+PHu9CsRI53CSVVI=" }, - "native-or-bluebird": { - "version": "1.1.2", - "resolved": "https://registry.npmjs.org/native-or-bluebird/-/native-or-bluebird-1.1.2.tgz", - "integrity": "sha1-OSHhECMtHreQ89rGG7NwUxx9NW4=", - "dev": true - }, "native-promise-only": { "version": "0.8.1", "resolved": "https://registry.npmjs.org/native-promise-only/-/native-promise-only-0.8.1.tgz", @@ -1652,14 +1669,14 @@ "optional": true }, "negotiator": { - "version": "0.6.1", - "resolved": "https://registry.npmjs.org/negotiator/-/negotiator-0.6.1.tgz", - "integrity": "sha1-KzJxhOiZIQEXeyhWP7XnECrNDKk=" + "version": "0.6.2", + "resolved": "https://registry.npmjs.org/negotiator/-/negotiator-0.6.2.tgz", + "integrity": "sha512-hZXc7K2e+PgeI1eDBe/10Ard4ekbfrrqG8Ep+8Jmf4JID2bNg7NvCPOZN+kfF574pFQI7mum2AUqDidoKqcTOw==" }, "node-fetch": { "version": "2.6.0", "resolved": "https://registry.npmjs.org/node-fetch/-/node-fetch-2.6.0.tgz", - "integrity": "sha512-8dG4H5ujfvFiqDmVu9fQ5bOHUC15JMjMY/Zumv26oOvvVJjM67KF8koCWIabKQ1GJIa9r2mMZscBq/TbdOcmNA==" + "integrity": "sha1-5jNFY4bUqlWGP2dqerDaqP3ssP0=" }, "node-forge": { "version": "0.8.5", @@ -1669,7 +1686,7 @@ "oauth-sign": { "version": "0.9.0", "resolved": "https://registry.npmjs.org/oauth-sign/-/oauth-sign-0.9.0.tgz", - "integrity": "sha1-R6ewFrqmi1+g7PPe4IqFxnmsZFU=" + "integrity": "sha512-fexhUFFPTGV8ybAtSIGbV6gOkSv8UtRbDBnAyLQw4QPKkgNlsH2ByPGtMUqdWkos6YCRmAqViwgZrJc/mRDzZQ==" }, "on-finished": { "version": "2.3.0", @@ -1680,9 +1697,9 @@ } }, "on-headers": { - "version": "1.0.1", - "resolved": "https://registry.npmjs.org/on-headers/-/on-headers-1.0.1.tgz", - "integrity": "sha1-ko9dD0cNSTQmUepnlLCFfBAGk/c=" + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/on-headers/-/on-headers-1.0.2.tgz", + "integrity": "sha512-pZAE+FJLoyITytdqK0U5s+FIpjN0JP3OzFi/u8Rx+EV5/W+JTWGXG8xFzevE7AjBfDqHv/8vL8qQsIhHnqRkrA==" }, "once": { "version": "1.4.0", @@ -1708,7 +1725,7 @@ "p-try": { "version": "2.2.0", "resolved": "https://registry.npmjs.org/p-try/-/p-try-2.2.0.tgz", - "integrity": "sha512-R4nPAVTAU0B9D35/Gk3uJf/7XYbQcyohSKdvAxIRSNghFl4e71hVoGnBNQz9cWaXxO2I10KTC+3jMdvvoKw6dQ==" + "integrity": "sha1-yyhoVA4xPWHeWPr741zpAE1VQOY=" }, "parse-duration": { "version": "0.1.2", @@ -1718,12 +1735,12 @@ "parse-ms": { "version": "2.1.0", "resolved": "https://registry.npmjs.org/parse-ms/-/parse-ms-2.1.0.tgz", - "integrity": "sha512-kHt7kzLoS9VBZfUsiKjv43mr91ea+U05EyKkEtqp7vNbHxmaVuEqN7XxeEVnGrMtYOAxGrDElSi96K7EgO1zCA==" + "integrity": "sha1-NIVlp1PUOR+lJAKZVrFyy3dTCX0=" }, "parseurl": { - "version": "1.3.2", - "resolved": "https://registry.npmjs.org/parseurl/-/parseurl-1.3.2.tgz", - "integrity": "sha1-/CidTtiZMRlGDBViUyYs3I3mW/M=" + "version": "1.3.3", + "resolved": "https://registry.npmjs.org/parseurl/-/parseurl-1.3.3.tgz", + "integrity": "sha512-CiyeOxFT/JZyN5m0z9PfXw4SCBJ6Sygz1Dpl0wqjlhDEGGBP1GnsUVEL0p63hoG1fcj3fHynXi9NYO4nWOL+qQ==" }, "path-is-absolute": { "version": "1.0.1", @@ -1733,7 +1750,7 @@ "path-parse": { "version": "1.0.6", "resolved": "https://registry.npmjs.org/path-parse/-/path-parse-1.0.6.tgz", - "integrity": "sha512-GSmOT2EbHrINBf9SR7CDELwlJ8AENk3Qn7OikK4nFYAu3Ote2+JYNVvkpAEQm3/TLNEJFD/xZJjzyxg3KBWOzw==" + "integrity": "sha1-1i27VnlAXXLEc37FhgDp3c8G0kw=" }, "path-to-regexp": { "version": "0.1.7", @@ -1748,7 +1765,7 @@ "pify": { "version": "4.0.1", "resolved": "https://registry.npmjs.org/pify/-/pify-4.0.1.tgz", - "integrity": "sha512-uB80kBFb/tfd68bVleG9T5GGsGPjJrLAUpR5PZIrhBnIaRTQRjqdJSsIKkOP6OAIFbj7GOrcudc5pNjZ+geV2g==" + "integrity": "sha1-SyzSXFDVmHNcUCkiJP2MbfQeMjE=" }, "policyfile": { "version": "0.0.4", @@ -1758,7 +1775,7 @@ "pretty-ms": { "version": "4.0.0", "resolved": "https://registry.npmjs.org/pretty-ms/-/pretty-ms-4.0.0.tgz", - "integrity": "sha512-qG66ahoLCwpLXD09ZPHSCbUWYTqdosB7SMP4OffgTgL2PBKXMuUsrk5Bwg8q4qPkjTXsKBMr+YK3Ltd/6F9s/Q==", + "integrity": "sha1-Mbr0G5T9AiJwmKqgO9YmCOsNbpI=", "requires": { "parse-ms": "^2.0.0" } @@ -1804,23 +1821,23 @@ } }, "proxy-addr": { - "version": "2.0.3", - "resolved": "https://registry.npmjs.org/proxy-addr/-/proxy-addr-2.0.3.tgz", - "integrity": "sha1-NV8mJQWmIWRrMTCnKOtkfiIFU0E=", + "version": "2.0.6", + "resolved": "https://registry.npmjs.org/proxy-addr/-/proxy-addr-2.0.6.tgz", + "integrity": "sha512-dh/frvCBVmSsDYzw6n926jv974gddhkFPfiN8hPOi30Wax25QZyZEGveluCgliBnqmuM+UJmBErbAUFIoDbjOw==", "requires": { "forwarded": "~0.1.2", - "ipaddr.js": "1.6.0" + "ipaddr.js": "1.9.1" } }, "psl": { - "version": "1.1.31", - "resolved": "https://registry.npmjs.org/psl/-/psl-1.1.31.tgz", - "integrity": "sha1-6aqG0BAbWxBcvpOsa3hM1UcnYYQ=" + "version": "1.8.0", + "resolved": "https://registry.npmjs.org/psl/-/psl-1.8.0.tgz", + "integrity": "sha512-RIdOzyoavK+hA18OGGWDqUTsCLhtA7IcZ/6NCs4fFJaHBDab+pDDmDIByWFRQJq2Cd7r1OoQxBGKOaztq+hjIQ==" }, "punycode": { - "version": "1.4.1", - "resolved": "https://registry.npmjs.org/punycode/-/punycode-1.4.1.tgz", - "integrity": "sha1-wNWmOycYgArY4esPpSachN1BhF4=" + "version": "2.1.1", + "resolved": "https://registry.npmjs.org/punycode/-/punycode-2.1.1.tgz", + "integrity": "sha512-XRsRjdf+j5ml+y/6GKHPZbrF/8p2Yga0JPtdqTIY2Xe5ohJPD9saDJJLPvp9+NSBprVvevdXZybnj2cv8OEd0A==" }, "q": { "version": "0.9.2", @@ -1828,9 +1845,9 @@ "integrity": "sha1-I8BsRsgTKGFqrhaNPuI6Vr1D2vY=" }, "qs": { - "version": "6.5.2", - "resolved": "https://registry.npmjs.org/qs/-/qs-6.5.2.tgz", - "integrity": "sha1-yzroBuh0BERYTvFUzo7pjUA/PjY=" + "version": "6.7.0", + "resolved": "https://registry.npmjs.org/qs/-/qs-6.7.0.tgz", + "integrity": "sha512-VCdBRNFTX1fyE7Nb6FYoURo/SPe62QCaAyzJvUjwRaIsc+NePBEniHlvxFmmX56+HZphIGtV0XeCirBtpDrTyQ==" }, "random-bytes": { "version": "1.0.0", @@ -1838,9 +1855,9 @@ "integrity": "sha1-T2ih3Arli9P7lYSMMDJNt11kNgs=" }, "range-parser": { - "version": "1.2.0", - "resolved": "https://registry.npmjs.org/range-parser/-/range-parser-1.2.0.tgz", - "integrity": "sha1-9JvmtIeJTdxA3MlKMi9hEJLgDV4=" + "version": "1.2.1", + "resolved": "https://registry.npmjs.org/range-parser/-/range-parser-1.2.1.tgz", + "integrity": "sha512-Hrgsx+orqoygnmhFbKaHE6c296J+HTAQXoxEF6gNupROmmGJRoyzfG3ccAveqCBrwr/2yxQ5BVd/GTl5agOwSg==" }, "raven": { "version": "1.1.3", @@ -1855,13 +1872,13 @@ } }, "raw-body": { - "version": "2.3.3", - "resolved": "https://registry.npmjs.org/raw-body/-/raw-body-2.3.3.tgz", - "integrity": "sha1-GzJOzmtXBuFThVvBFIxlu39uoMM=", + "version": "2.4.0", + "resolved": "https://registry.npmjs.org/raw-body/-/raw-body-2.4.0.tgz", + "integrity": "sha512-4Oz8DUIwdvoa5qMJelxipzi/iJIi40O5cGV1wNYp5hvZP8ZN0T+jiNkL0QepXs+EsQ9XJ8ipEDoiH70ySUJP3Q==", "requires": { - "bytes": "3.0.0", - "http-errors": "1.6.3", - "iconv-lite": "0.4.23", + "bytes": "3.1.0", + "http-errors": "1.7.2", + "iconv-lite": "0.4.24", "unpipe": "1.0.0" } }, @@ -1919,13 +1936,13 @@ } }, "redis-sharelatex": { - "version": "1.0.12", - "resolved": "https://registry.npmjs.org/redis-sharelatex/-/redis-sharelatex-1.0.12.tgz", - "integrity": "sha512-Z+LDGaRNgZ+NiDaCC/R0N3Uy6SCtbKXqiXlvCwAbIQRSZUc69OVx/cQ3i5qDF7zeERhh+pnTd+zGs8nVfa5p+Q==", + "version": "1.0.13", + "resolved": "https://registry.npmjs.org/redis-sharelatex/-/redis-sharelatex-1.0.13.tgz", + "integrity": "sha512-sAQNofqfcMlIxzxNJF1qUspJKDM1VuuIOrGZQX9nb5JtcJ5cusa5sc+Oyb51eymPV5mZGWT3u07tKtv4jdXVIg==", "requires": { "async": "^2.5.0", "coffee-script": "1.8.0", - "ioredis": "~4.16.1", + "ioredis": "~4.17.3", "redis-sentinel": "0.1.1", "underscore": "1.7.0" }, @@ -1954,9 +1971,9 @@ } }, "request": { - "version": "2.88.0", - "resolved": "https://registry.npmjs.org/request/-/request-2.88.0.tgz", - "integrity": "sha1-nC/KT301tZLv5Xx/ClXoEFIST+8=", + "version": "2.88.2", + "resolved": "https://registry.npmjs.org/request/-/request-2.88.2.tgz", + "integrity": "sha512-MsvtOrfG9ZcrOwAW+Qi+F6HbD0CWXEh9ou77uOb7FM2WPhwT7smM833PzanhJLsgXjN89Ir6V2PczXNnMpwKhw==", "requires": { "aws-sign2": "~0.7.0", "aws4": "^1.8.0", @@ -1965,7 +1982,7 @@ "extend": "~3.0.2", "forever-agent": "~0.6.1", "form-data": "~2.3.2", - "har-validator": "~5.1.0", + "har-validator": "~5.1.3", "http-signature": "~1.2.0", "is-typedarray": "~1.0.0", "isstream": "~0.1.2", @@ -1975,33 +1992,47 @@ "performance-now": "^2.1.0", "qs": "~6.5.2", "safe-buffer": "^5.1.2", - "tough-cookie": "~2.4.3", + "tough-cookie": "~2.5.0", "tunnel-agent": "^0.6.0", "uuid": "^3.3.2" }, "dependencies": { "mime-db": { - "version": "1.38.0", - "resolved": "https://registry.npmjs.org/mime-db/-/mime-db-1.38.0.tgz", - "integrity": "sha1-GiqrFtqesWe0nG5N8tnGjWPY4q0=" + "version": "1.44.0", + "resolved": "https://registry.npmjs.org/mime-db/-/mime-db-1.44.0.tgz", + "integrity": "sha512-/NOTfLrsPBVeH7YtFPgsVWveuL+4SjjYxaQ1xtM1KMFj7HdxlBlxeyNLzhyJVx7r4rZGJAZ/6lkKCitSc/Nmpg==" }, "mime-types": { - "version": "2.1.22", - "resolved": "https://registry.npmjs.org/mime-types/-/mime-types-2.1.22.tgz", - "integrity": "sha1-/ms1WhkJJqt2mMmgVWoRGZshmb0=", + "version": "2.1.27", + "resolved": "https://registry.npmjs.org/mime-types/-/mime-types-2.1.27.tgz", + "integrity": "sha512-JIhqnCasI9yD+SsmkquHBxTSEuZdQX5BuQnS2Vc7puQQQ+8yiP5AY5uWhpdv4YL4VM5c6iliiYWPgJ/nJQLp7w==", "requires": { - "mime-db": "~1.38.0" + "mime-db": "1.44.0" } }, + "qs": { + "version": "6.5.2", + "resolved": "https://registry.npmjs.org/qs/-/qs-6.5.2.tgz", + "integrity": "sha512-N5ZAX4/LxJmF+7wN74pUD6qAh9/wnvdQcjq9TZjevvXzSUo7bfmw91saqMjzGS2xq91/odN2dW/WOl7qQHNDGA==" + }, "safe-buffer": { - "version": "5.1.2", - "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.1.2.tgz", - "integrity": "sha1-mR7GnSluAxN0fVm9/St0XDX4go0=" + "version": "5.2.1", + "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.2.1.tgz", + "integrity": "sha512-rp3So07KcdmmKbGvgaNxQSJr7bGVSVk5S9Eq1F+ppbRo70+YeaDxkw5Dd8NPN+GD6bjnYm2VuPuCXmpuYvmCXQ==" + }, + "tough-cookie": { + "version": "2.5.0", + "resolved": "https://registry.npmjs.org/tough-cookie/-/tough-cookie-2.5.0.tgz", + "integrity": "sha512-nlLsUzgm1kfLXSXfRZMc1KLAugd4hqJHDTvc2hDIwS3mZAfMEuMbc03SujMF+GEcpaX/qboeycw6iO8JwVv2+g==", + "requires": { + "psl": "^1.1.28", + "punycode": "^2.1.1" + } }, "uuid": { - "version": "3.3.2", - "resolved": "https://registry.npmjs.org/uuid/-/uuid-3.3.2.tgz", - "integrity": "sha1-G0r0lV6zB3xQHCOHL8ZROBFYcTE=" + "version": "3.4.0", + "resolved": "https://registry.npmjs.org/uuid/-/uuid-3.4.0.tgz", + "integrity": "sha512-HjSDRw6gZE5JMggctHBcjVak08+KEVhSIiDzFnT9S9aegmp85S/bReBVTb4QTFaRNptJ9kuYaNhnbNEOkbKb/A==" } } }, @@ -2047,7 +2078,7 @@ "retry-axios": { "version": "0.3.2", "resolved": "https://registry.npmjs.org/retry-axios/-/retry-axios-0.3.2.tgz", - "integrity": "sha512-jp4YlI0qyDFfXiXGhkCOliBN1G7fRH03Nqy8YdShzGqbY5/9S2x/IR6C88ls2DFkbWuL3ASkP7QD3pVrNpPgwQ==" + "integrity": "sha1-V1fID1hbTMTEmGqi/9R6YMbTXhM=" }, "retry-request": { "version": "4.1.1", @@ -2096,7 +2127,7 @@ "safer-buffer": { "version": "2.1.2", "resolved": "https://registry.npmjs.org/safer-buffer/-/safer-buffer-2.1.2.tgz", - "integrity": "sha1-RPoWGwGHuVSd2Eu5GAL5vYOFzWo=" + "integrity": "sha512-YZo3K82SD7Riyi0E1EQPojLz7kpepnSQI9IyPbHHg1XXXevb5dJI7tpyN2ADxGcQbHG7vcyRHk0cbwqcQriUtg==" }, "samsam": { "version": "1.3.0", @@ -2128,9 +2159,9 @@ "integrity": "sha512-b39TBaTSfV6yBrapU89p5fKekE2m/NwnDocOVruQFS1/veMgdzuPcnOM34M6CwxW8jH/lxEa5rBoDeUwu5HHTw==" }, "send": { - "version": "0.16.2", - "resolved": "https://registry.npmjs.org/send/-/send-0.16.2.tgz", - "integrity": "sha1-bsyh4PjBVtFBWXVZhI32RzCmu8E=", + "version": "0.17.1", + "resolved": "https://registry.npmjs.org/send/-/send-0.17.1.tgz", + "integrity": "sha512-BsVKsiGcQMFwT8UxypobUKyv7irCNRHk1T0G680vk88yf6LBByGcZJOTJCrTP2xVN6yI+XjPJcNuE3V4fT9sAg==", "requires": { "debug": "2.6.9", "depd": "~1.1.2", @@ -2139,36 +2170,36 @@ "escape-html": "~1.0.3", "etag": "~1.8.1", "fresh": "0.5.2", - "http-errors": "~1.6.2", - "mime": "1.4.1", - "ms": "2.0.0", + "http-errors": "~1.7.2", + "mime": "1.6.0", + "ms": "2.1.1", "on-finished": "~2.3.0", - "range-parser": "~1.2.0", - "statuses": "~1.4.0" + "range-parser": "~1.2.1", + "statuses": "~1.5.0" }, "dependencies": { - "statuses": { - "version": "1.4.0", - "resolved": "https://registry.npmjs.org/statuses/-/statuses-1.4.0.tgz", - "integrity": "sha1-u3PURtonlhBu/MG2AaJT1sRr0Ic=" + "ms": { + "version": "2.1.1", + "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.1.tgz", + "integrity": "sha512-tgp+dl5cGk28utYktBsrFqA7HKgrhgPsg6Z/EfhWI4gl1Hwq8B/GmY/0oXZ6nF8hDVesS/FpnYaD/kOWhYQvyg==" } } }, "serve-static": { - "version": "1.13.2", - "resolved": "https://registry.npmjs.org/serve-static/-/serve-static-1.13.2.tgz", - "integrity": "sha1-CV6Ecv1bRiN9tQzkhqQ/S4bGzsE=", + "version": "1.14.1", + "resolved": "https://registry.npmjs.org/serve-static/-/serve-static-1.14.1.tgz", + "integrity": "sha512-JMrvUwE54emCYWlTI+hGrGv5I8dEwmco/00EvkzIIsR7MqrHonbD9pO2MOfFnpFntl7ecpZs+3mW+XbQZu9QCg==", "requires": { "encodeurl": "~1.0.2", "escape-html": "~1.0.3", - "parseurl": "~1.3.2", - "send": "0.16.2" + "parseurl": "~1.3.3", + "send": "0.17.1" } }, "setprototypeof": { - "version": "1.1.0", - "resolved": "https://registry.npmjs.org/setprototypeof/-/setprototypeof-1.1.0.tgz", - "integrity": "sha1-0L2FU2iHtv58DYGMuWLZ2RxU5lY=" + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/setprototypeof/-/setprototypeof-1.1.1.tgz", + "integrity": "sha512-JvdAWfbXeIGaZ9cILp38HntZSFSo3mWg6xGcJJsd+d4aRMOqauag1C63dJfDw7OaMYwEbHMOxEZ1lqVRYP2OAw==" }, "settings-sharelatex": { "version": "1.1.0", @@ -2188,7 +2219,7 @@ "shimmer": { "version": "1.2.1", "resolved": "https://registry.npmjs.org/shimmer/-/shimmer-1.2.1.tgz", - "integrity": "sha512-sQTKC1Re/rM6XyFM6fIAGHRPVGvyXfgzIDvzoq608vM+jeyVD0Tu1E6Np0Kc2zAIFWIj963V2800iF/9LPieQw==" + "integrity": "sha1-YQhZ994ye1h+/r9QH7QxF/mv8zc=" }, "sinon": { "version": "2.4.1", @@ -2273,12 +2304,12 @@ "source-map": { "version": "0.6.1", "resolved": "https://registry.npmjs.org/source-map/-/source-map-0.6.1.tgz", - "integrity": "sha512-UjgapumWlbMhkBgzT7Ykc5YXUT46F0iKu8SGXq0bcwP5dz/h0Plj6enJqjz1Zbq2l5WaqYnrVbwWOWMyF3F47g==" + "integrity": "sha1-dHIq8y6WFOnCh6jQu95IteLxomM=" }, "split": { "version": "1.0.1", "resolved": "https://registry.npmjs.org/split/-/split-1.0.1.tgz", - "integrity": "sha512-mTyOoPbrivtXnwnIxZRFYRrPNtEFKlpB2fvjSnCQUiAA6qAZzqwna5envK4uk6OIeP17CsdF3rSBGYVBsU0Tkg==", + "integrity": "sha1-YFvZvjA6pZ+zX5Ip++oN3snqB9k=", "requires": { "through": "2" } @@ -2286,7 +2317,7 @@ "sshpk": { "version": "1.16.1", "resolved": "https://registry.npmjs.org/sshpk/-/sshpk-1.16.1.tgz", - "integrity": "sha1-+2YcC+8ps520B2nuOfpwCT1vaHc=", + "integrity": "sha512-HXXqVUq7+pcKeLqqZj6mHFUMvXtOJt1uoUx09pFW6011inTMxqI8BA8PM95myrIyyKwdnzjdFjLiE6KBPVtJIg==", "requires": { "asn1": "~0.2.3", "assert-plus": "^1.0.0", @@ -2297,21 +2328,6 @@ "jsbn": "~0.1.0", "safer-buffer": "^2.0.2", "tweetnacl": "~0.14.0" - }, - "dependencies": { - "asn1": { - "version": "0.2.4", - "resolved": "https://registry.npmjs.org/asn1/-/asn1-0.2.4.tgz", - "integrity": "sha1-jSR136tVO7M+d7VOWeiAu4ziMTY=", - "requires": { - "safer-buffer": "~2.1.0" - } - }, - "assert-plus": { - "version": "1.0.0", - "resolved": "https://registry.npmjs.org/assert-plus/-/assert-plus-1.0.0.tgz", - "integrity": "sha1-8S4PPF13sLHN2RRpQuTpbB5N1SU=" - } } }, "stack-trace": { @@ -2342,7 +2358,7 @@ "string_decoder": { "version": "1.1.1", "resolved": "https://registry.npmjs.org/string_decoder/-/string_decoder-1.1.1.tgz", - "integrity": "sha512-n/ShnvDi6FHbbVfviro+WojiFzv+s8MPMHBczVePfUpDJLwoLT0ht1l4YwBCbi8pJAveEEdnkHyPyTP/mzRfwg==", + "integrity": "sha1-nPFhG6YmhdcDCunkujQUnDrwP8g=", "requires": { "safe-buffer": "~5.1.0" } @@ -2358,7 +2374,7 @@ "teeny-request": { "version": "3.11.3", "resolved": "https://registry.npmjs.org/teeny-request/-/teeny-request-3.11.3.tgz", - "integrity": "sha512-CKncqSF7sH6p4rzCgkb/z/Pcos5efl0DmolzvlqRQUNcpRIruOhY9+T1FsIlyEbfWd7MsFpodROOwHYh2BaXzw==", + "integrity": "sha1-M1xin3ZF5dZZk2LfLzIwxMvCOlU=", "requires": { "https-proxy-agent": "^2.2.1", "node-fetch": "^2.2.0", @@ -2402,13 +2418,25 @@ "resolved": "https://registry.npmjs.org/tinycolor/-/tinycolor-0.0.1.tgz", "integrity": "sha1-MgtaUtg6u1l42Bo+iH1K77FaYWQ=" }, + "toidentifier": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/toidentifier/-/toidentifier-1.0.0.tgz", + "integrity": "sha512-yaOH/Pk/VEhBWWTlhI+qXxDFXlejDGcQipMlyxda9nthulaxLZUNcUqFxokp0vcYnvteJln5FNQDRrxj3YcbVw==" + }, "tough-cookie": { "version": "2.4.3", "resolved": "https://registry.npmjs.org/tough-cookie/-/tough-cookie-2.4.3.tgz", - "integrity": "sha1-U/Nto/R3g7CSWvoG/587FlKA94E=", + "integrity": "sha512-Q5srk/4vDM54WJsJio3XNn6K2sCG+CQ8G5Wz6bZhRZoAe/+TxjWB/GlFAnYEbkYVlON9FMk/fE3h2RLpPXo4lQ==", "requires": { "psl": "^1.1.24", "punycode": "^1.4.1" + }, + "dependencies": { + "punycode": { + "version": "1.4.1", + "resolved": "https://registry.npmjs.org/punycode/-/punycode-1.4.1.tgz", + "integrity": "sha1-wNWmOycYgArY4esPpSachN1BhF4=" + } } }, "tunnel-agent": { @@ -2431,12 +2459,27 @@ "dev": true }, "type-is": { - "version": "1.6.16", - "resolved": "https://registry.npmjs.org/type-is/-/type-is-1.6.16.tgz", - "integrity": "sha1-+JzjQVQcZysl7nrjxz3uOyvlAZQ=", + "version": "1.6.18", + "resolved": "https://registry.npmjs.org/type-is/-/type-is-1.6.18.tgz", + "integrity": "sha512-TkRKr9sUTxEH8MdfuCSP7VizJyzRNMjj2J2do2Jr3Kym598JVdEksuzPQCnlFPW4ky9Q+iA+ma9BGm06XQBy8g==", "requires": { "media-typer": "0.3.0", - "mime-types": "~2.1.18" + "mime-types": "~2.1.24" + }, + "dependencies": { + "mime-db": { + "version": "1.44.0", + "resolved": "https://registry.npmjs.org/mime-db/-/mime-db-1.44.0.tgz", + "integrity": "sha512-/NOTfLrsPBVeH7YtFPgsVWveuL+4SjjYxaQ1xtM1KMFj7HdxlBlxeyNLzhyJVx7r4rZGJAZ/6lkKCitSc/Nmpg==" + }, + "mime-types": { + "version": "2.1.27", + "resolved": "https://registry.npmjs.org/mime-types/-/mime-types-2.1.27.tgz", + "integrity": "sha512-JIhqnCasI9yD+SsmkquHBxTSEuZdQX5BuQnS2Vc7puQQQ+8yiP5AY5uWhpdv4YL4VM5c6iliiYWPgJ/nJQLp7w==", + "requires": { + "mime-db": "1.44.0" + } + } } }, "uglify-js": { @@ -2445,13 +2488,11 @@ "integrity": "sha1-tULCx29477NLIAsgF3Y0Mw/3ArY=" }, "uid-safe": { - "version": "1.1.0", - "resolved": "https://registry.npmjs.org/uid-safe/-/uid-safe-1.1.0.tgz", - "integrity": "sha1-WNbF2r+N+9jVKDSDmAbAP9YUMjI=", - "dev": true, + "version": "2.1.5", + "resolved": "https://registry.npmjs.org/uid-safe/-/uid-safe-2.1.5.tgz", + "integrity": "sha512-KPHm4VL5dDXKz01UuEd88Df+KzynaohSL9fBh096KWAxSKZQDI2uBrVqtvRM4rwrIrRRKsdLNML/lnaaVSRioA==", "requires": { - "base64-url": "1.2.1", - "native-or-bluebird": "~1.1.2" + "random-bytes": "~1.0.0" } }, "underscore": { @@ -2467,16 +2508,9 @@ "uri-js": { "version": "4.2.2", "resolved": "https://registry.npmjs.org/uri-js/-/uri-js-4.2.2.tgz", - "integrity": "sha1-lMVA4f93KVbiKZUHwBCupsiDjrA=", + "integrity": "sha512-KY9Frmirql91X2Qgjry0Wd4Y+YTdrdZheS8TFwvkbLWf/G5KNJDCh6pKL5OZctEW4+0Baa5idK2ZQuELRwPznQ==", "requires": { "punycode": "^2.1.0" - }, - "dependencies": { - "punycode": { - "version": "2.1.1", - "resolved": "https://registry.npmjs.org/punycode/-/punycode-2.1.1.tgz", - "integrity": "sha1-tYsBCsQMIsVldhbI0sLALHv0eew=" - } } }, "util-deprecate": { @@ -2507,13 +2541,6 @@ "assert-plus": "^1.0.0", "core-util-is": "1.0.2", "extsprintf": "^1.2.0" - }, - "dependencies": { - "assert-plus": { - "version": "1.0.0", - "resolved": "https://registry.npmjs.org/assert-plus/-/assert-plus-1.0.0.tgz", - "integrity": "sha1-8S4PPF13sLHN2RRpQuTpbB5N1SU=" - } } }, "wrappy": { diff --git a/services/real-time/package.json b/services/real-time/package.json index b50fb9c6ce..157895473e 100644 --- a/services/real-time/package.json +++ b/services/real-time/package.json @@ -24,15 +24,15 @@ "async": "^0.9.0", "base64id": "0.1.0", "basic-auth-connect": "^1.0.0", - "body-parser": "^1.12.0", + "body-parser": "^1.19.0", "connect-redis": "^2.1.0", - "cookie-parser": "^1.3.3", - "express": "^4.10.1", - "express-session": "^1.9.1", + "cookie-parser": "^1.4.5", + "express": "^4.17.1", + "express-session": "^1.17.1", "logger-sharelatex": "^1.7.0", "metrics-sharelatex": "^2.6.2", - "redis-sharelatex": "^1.0.12", - "request": "^2.88.0", + "redis-sharelatex": "^1.0.13", + "request": "^2.88.2", "settings-sharelatex": "^1.1.0", "socket.io": "0.9.19", "socket.io-client": "^0.9.16" @@ -40,11 +40,11 @@ "devDependencies": { "bunyan": "~0.22.3", "chai": "~1.9.1", - "cookie-signature": "^1.0.5", + "cookie-signature": "^1.1.0", "sandboxed-module": "~0.3.0", "sinon": "^2.4.1", "mocha": "^4.0.1", - "uid-safe": "^1.0.1", + "uid-safe": "^2.1.5", "timekeeper": "0.0.4" } } From ce3266c59ccaa891113828548af09593c58f787c Mon Sep 17 00:00:00 2001 From: Miguel Serrano Date: Fri, 19 Jun 2020 17:29:31 +0200 Subject: [PATCH 357/491] Fixed sha downgrades --- services/real-time/package-lock.json | 450 +++++++++++++-------------- 1 file changed, 225 insertions(+), 225 deletions(-) diff --git a/services/real-time/package-lock.json b/services/real-time/package-lock.json index 94141c4e6c..72c96d7df6 100644 --- a/services/real-time/package-lock.json +++ b/services/real-time/package-lock.json @@ -7,7 +7,7 @@ "@google-cloud/common": { "version": "0.32.1", "resolved": "https://registry.npmjs.org/@google-cloud/common/-/common-0.32.1.tgz", - "integrity": "sha1-ajLDQBcs6j22Z00ODjTnh0CgBz8=", + "integrity": "sha512-bLdPzFvvBMtVkwsoBtygE9oUm3yrNmPa71gvOgucYI/GqvNP2tb6RYsDHPq98kvignhcgHGDI5wyNgxaCo8bKQ==", "requires": { "@google-cloud/projectify": "^0.3.3", "@google-cloud/promisify": "^0.4.0", @@ -25,7 +25,7 @@ "@google-cloud/debug-agent": { "version": "3.2.0", "resolved": "https://registry.npmjs.org/@google-cloud/debug-agent/-/debug-agent-3.2.0.tgz", - "integrity": "sha1-2qdjWhaYpWY31dxXzhED536uKdM=", + "integrity": "sha512-fP87kYbS6aeDna08BivwQ1J260mwJGchRi99XdWCgqbRwuFac8ul0OT5i2wEeDSc5QaDX8ZuWQQ0igZvh1rTyQ==", "requires": { "@google-cloud/common": "^0.32.0", "@sindresorhus/is": "^0.15.0", @@ -53,7 +53,7 @@ "@google-cloud/profiler": { "version": "0.2.3", "resolved": "https://registry.npmjs.org/@google-cloud/profiler/-/profiler-0.2.3.tgz", - "integrity": "sha1-Fj3738Mwuug1X+RuHlvgZTV7H1w=", + "integrity": "sha512-rNvtrFtIebIxZEJ/O0t8n7HciZGIXBo8DvHxWqAmsCaeLvkTtsaL6HmPkwxrNQ1IhbYWAxF+E/DwCiHyhKmgTg==", "requires": { "@google-cloud/common": "^0.26.0", "@types/console-log-level": "^1.4.0", @@ -75,7 +75,7 @@ "@google-cloud/common": { "version": "0.26.2", "resolved": "https://registry.npmjs.org/@google-cloud/common/-/common-0.26.2.tgz", - "integrity": "sha1-nFTiRxqEqgMelaJIJJduCA8lVkU=", + "integrity": "sha512-xJ2M/q3MrUbnYZuFlpF01caAlEhAUoRn0NXp93Hn3pkFpfSOG8YfbKbpBAHvcKVbBOAKVIwPsleNtuyuabUwLQ==", "requires": { "@google-cloud/projectify": "^0.3.2", "@google-cloud/promisify": "^0.3.0", @@ -94,17 +94,17 @@ "@google-cloud/promisify": { "version": "0.3.1", "resolved": "https://registry.npmjs.org/@google-cloud/promisify/-/promisify-0.3.1.tgz", - "integrity": "sha1-9kHm2USo4KBe4MsQkd+mAIm+zbo=" + "integrity": "sha512-QzB0/IMvB0eFxFK7Eqh+bfC8NLv3E9ScjWQrPOk6GgfNroxcVITdTlT8NRsRrcp5+QQJVPLkRqKG0PUdaWXmHw==" }, "arrify": { "version": "1.0.1", "resolved": "https://registry.npmjs.org/arrify/-/arrify-1.0.1.tgz", - "integrity": "sha1-iYUI2iIm84DfkEcoRWhJwVAaSw0=" + "integrity": "sha512-3CYzex9M9FGQjCGMGyi6/31c8GJbgb0qGyrx5HWxPd0aCwh4cB2YjMb2Xf9UuoogrMrlO9cTqnB5rI5GHZTcUA==" }, "gcp-metadata": { "version": "0.9.3", "resolved": "https://registry.npmjs.org/gcp-metadata/-/gcp-metadata-0.9.3.tgz", - "integrity": "sha1-H510lfdGChRSZIHynhFZbdVj3SY=", + "integrity": "sha512-caV4S84xAjENtpezLCT/GILEAF5h/bC4cNqZFmt/tjTn8t+JBtTkQrgBrJu3857YdsnlM8rxX/PMcKGtE8hUlw==", "requires": { "gaxios": "^1.0.2", "json-bigint": "^0.3.0" @@ -113,7 +113,7 @@ "google-auth-library": { "version": "2.0.2", "resolved": "https://registry.npmjs.org/google-auth-library/-/google-auth-library-2.0.2.tgz", - "integrity": "sha1-ejFdIDZ0Svavyth7IQ7mY4tA9Xs=", + "integrity": "sha512-FURxmo1hBVmcfLauuMRKOPYAPKht3dGuI2wjeJFalDUThO0HoYVjr4yxt5cgYSFm1dgUpmN9G/poa7ceTFAIiA==", "requires": { "axios": "^0.18.0", "gcp-metadata": "^0.7.0", @@ -127,7 +127,7 @@ "gcp-metadata": { "version": "0.7.0", "resolved": "https://registry.npmjs.org/gcp-metadata/-/gcp-metadata-0.7.0.tgz", - "integrity": "sha1-bDXbtSvaMqQnu5yY9UI33dG1QG8=", + "integrity": "sha512-ffjC09amcDWjh3VZdkDngIo7WoluyC5Ag9PAYxZbmQLOLNI8lvPtoKTSCyU54j2gwy5roZh6sSMTfkY2ct7K3g==", "requires": { "axios": "^0.18.0", "extend": "^3.0.1", @@ -146,17 +146,17 @@ "@google-cloud/projectify": { "version": "0.3.3", "resolved": "https://registry.npmjs.org/@google-cloud/projectify/-/projectify-0.3.3.tgz", - "integrity": "sha1-vekQPVCyCj6jM334xng6dm5w1B0=" + "integrity": "sha512-7522YHQ4IhaafgSunsFF15nG0TGVmxgXidy9cITMe+256RgqfcrfWphiMufW+Ou4kqagW/u3yxwbzVEW3dk2Uw==" }, "@google-cloud/promisify": { "version": "0.4.0", "resolved": "https://registry.npmjs.org/@google-cloud/promisify/-/promisify-0.4.0.tgz", - "integrity": "sha1-T7/PTYW7ai5MzwWqY9KxDWyarZs=" + "integrity": "sha512-4yAHDC52TEMCNcMzVC8WlqnKKKq+Ssi2lXoUg9zWWkZ6U6tq9ZBRYLHHCRdfU+EU9YJsVmivwGcKYCjRGjnf4Q==" }, "@google-cloud/trace-agent": { "version": "3.6.1", "resolved": "https://registry.npmjs.org/@google-cloud/trace-agent/-/trace-agent-3.6.1.tgz", - "integrity": "sha1-W+dEE5TQ6ldY8o25IqUAT/PwO+w=", + "integrity": "sha512-KDo85aPN4gSxJ7oEIOlKd7aGENZFXAM1kbIn1Ds+61gh/K1CQWSyepgJo3nUpAwH6D1ezDWV7Iaf8ueoITc8Uw==", "requires": { "@google-cloud/common": "^0.32.1", "builtin-modules": "^3.0.0", @@ -183,27 +183,27 @@ "@protobufjs/aspromise": { "version": "1.1.2", "resolved": "https://registry.npmjs.org/@protobufjs/aspromise/-/aspromise-1.1.2.tgz", - "integrity": "sha1-m4sMxmPWaafY9vXQiToU00jzD78=" + "integrity": "sha512-j+gKExEuLmKwvz3OgROXtrJ2UG2x8Ch2YZUxahh+s1F2HZ+wAceUNLkvy6zKCPVRkU++ZWQrdxsUeQXmcg4uoQ==" }, "@protobufjs/base64": { "version": "1.1.2", "resolved": "https://registry.npmjs.org/@protobufjs/base64/-/base64-1.1.2.tgz", - "integrity": "sha1-TIVzDlm5ofHzSQR9vyQpYDS7JzU=" + "integrity": "sha512-AZkcAA5vnN/v4PDqKyMR5lx7hZttPDgClv83E//FMNhR2TMcLUhfRUBHCmSl0oi9zMgDDqRUJkSxO3wm85+XLg==" }, "@protobufjs/codegen": { "version": "2.0.4", "resolved": "https://registry.npmjs.org/@protobufjs/codegen/-/codegen-2.0.4.tgz", - "integrity": "sha1-fvN/DQEPsCitGtWXIuUG2SYoFcs=" + "integrity": "sha512-YyFaikqM5sH0ziFZCN3xDC7zeGaB/d0IUb9CATugHWbd1FRFwWwt4ld4OYMPWu5a3Xe01mGAULCdqhMlPl29Jg==" }, "@protobufjs/eventemitter": { "version": "1.1.0", "resolved": "https://registry.npmjs.org/@protobufjs/eventemitter/-/eventemitter-1.1.0.tgz", - "integrity": "sha1-NVy8mLr61ZePntCV85diHx0Ga3A=" + "integrity": "sha512-j9ednRT81vYJ9OfVuXG6ERSTdEL1xVsNgqpkxMsbIabzSo3goCjDIveeGv5d03om39ML71RdmrGNjG5SReBP/Q==" }, "@protobufjs/fetch": { "version": "1.1.0", "resolved": "https://registry.npmjs.org/@protobufjs/fetch/-/fetch-1.1.0.tgz", - "integrity": "sha1-upn7WYYUr2VwDBYZ/wbUVLDYTEU=", + "integrity": "sha512-lljVXpqXebpsijW71PZaCYeIcE5on1w5DlQy5WH6GLbFryLUrBD4932W/E2BSpfRJWseIL4v/KPgBFxDOIdKpQ==", "requires": { "@protobufjs/aspromise": "^1.1.1", "@protobufjs/inquire": "^1.1.0" @@ -212,47 +212,47 @@ "@protobufjs/float": { "version": "1.0.2", "resolved": "https://registry.npmjs.org/@protobufjs/float/-/float-1.0.2.tgz", - "integrity": "sha1-Xp4avctz/Ap8uLKR33jIy9l7h9E=" + "integrity": "sha512-Ddb+kVXlXst9d+R9PfTIxh1EdNkgoRe5tOX6t01f1lYWOvJnSPDBlG241QLzcyPdoNTsblLUdujGSE4RzrTZGQ==" }, "@protobufjs/inquire": { "version": "1.1.0", "resolved": "https://registry.npmjs.org/@protobufjs/inquire/-/inquire-1.1.0.tgz", - "integrity": "sha1-/yAOPnzyQp4tyvwRQIKOjMY48Ik=" + "integrity": "sha512-kdSefcPdruJiFMVSbn801t4vFK7KB/5gd2fYvrxhuJYg8ILrmn9SKSX2tZdV6V+ksulWqS7aXjBcRXl3wHoD9Q==" }, "@protobufjs/path": { "version": "1.1.2", "resolved": "https://registry.npmjs.org/@protobufjs/path/-/path-1.1.2.tgz", - "integrity": "sha1-bMKyDFya1q0NzP0hynZz2Nf79o0=" + "integrity": "sha512-6JOcJ5Tm08dOHAbdR3GrvP+yUUfkjG5ePsHYczMFLq3ZmMkAD98cDgcT2iA1lJ9NVwFd4tH/iSSoe44YWkltEA==" }, "@protobufjs/pool": { "version": "1.1.0", "resolved": "https://registry.npmjs.org/@protobufjs/pool/-/pool-1.1.0.tgz", - "integrity": "sha1-Cf0V8tbTq/qbZbw2ZQbWrXhG/1Q=" + "integrity": "sha512-0kELaGSIDBKvcgS4zkjz1PeddatrjYcmMWOlAuAPwAeccUrPHdUqo/J6LiymHHEiJT5NrF1UVwxY14f+fy4WQw==" }, "@protobufjs/utf8": { "version": "1.1.0", "resolved": "https://registry.npmjs.org/@protobufjs/utf8/-/utf8-1.1.0.tgz", - "integrity": "sha1-p3c2C1s5oaLlEG+OhY8v0tBgxXA=" + "integrity": "sha512-Vvn3zZrhQZkkBE8LSuW3em98c0FwgO4nxzv6OdSxPKJIEKY2bGbHn+mhGIPerzI4twdxaP8/0+06HBpwf345Lw==" }, "@sindresorhus/is": { "version": "0.15.0", "resolved": "https://registry.npmjs.org/@sindresorhus/is/-/is-0.15.0.tgz", - "integrity": "sha1-lpFbqgXmpqHRN7rfSYTT/AWCC7Y=" + "integrity": "sha512-lu8BpxjAtRCAo5ifytTpCPCj99LF7o/2Myn+NXyNCBqvPYn7Pjd76AMmUB5l7XF1U6t0hcWrlEM5ESufW7wAeA==" }, "@types/caseless": { "version": "0.12.2", "resolved": "https://registry.npmjs.org/@types/caseless/-/caseless-0.12.2.tgz", - "integrity": "sha1-9l09Y4ngHutFi9VNyPUrlalGO8g=" + "integrity": "sha512-6ckxMjBBD8URvjB6J3NcnuAn5Pkl7t3TizAg+xdlzzQGSPSmBcXf8KoIH0ua/i+tio+ZRUHEXp0HEmvaR4kt0w==" }, "@types/console-log-level": { "version": "1.4.0", "resolved": "https://registry.npmjs.org/@types/console-log-level/-/console-log-level-1.4.0.tgz", - "integrity": "sha1-7/ccQa689RyLpa2LBdfVQkviuPM=" + "integrity": "sha512-x+OscEQwcx5Biair4enH7ov9W+clcqUWaZRaxn5IkT4yNWWjRr2oiYDkY/x1uXSTVZOQ2xlbFQySaQGB+VdXGQ==" }, "@types/duplexify": { "version": "3.6.0", "resolved": "https://registry.npmjs.org/@types/duplexify/-/duplexify-3.6.0.tgz", - "integrity": "sha1-38grZL06IWj1vSZESvFlvwI33Ng=", + "integrity": "sha512-5zOA53RUlzN74bvrSGwjudssD9F3a797sDZQkiYpUOxW+WHaXTCPz4/d5Dgi6FKnOqZ2CpaTo0DhgIfsXAOE/A==", "requires": { "@types/node": "*" } @@ -293,7 +293,7 @@ "@types/semver": { "version": "5.5.0", "resolved": "https://registry.npmjs.org/@types/semver/-/semver-5.5.0.tgz", - "integrity": "sha1-FGwqKe59O65L8vyydGNuJkyBPEU=" + "integrity": "sha512-41qEJgBH/TWgo5NFSvBCJ1qkoi3Q6ONSF2avrHq1LVEZfYpdHmj0y9SuTK+u9ZhG1sYQKBL1AWXKyLWP4RaUoQ==" }, "@types/tough-cookie": { "version": "2.3.6", @@ -303,7 +303,7 @@ "abort-controller": { "version": "3.0.0", "resolved": "https://registry.npmjs.org/abort-controller/-/abort-controller-3.0.0.tgz", - "integrity": "sha1-6vVNU7YrrkE46AnKIlyEOabvs5I=", + "integrity": "sha512-h8lQ8tacZYnR3vNQTgibj+tODHI5/+l06Au2Pcriv/Gmet0eaj4TwWH41sO9wnHDiQsEj19q0drzdWdeAHtweg==", "requires": { "event-target-shim": "^5.0.0" } @@ -340,7 +340,7 @@ "active-x-obfuscator": { "version": "0.0.1", "resolved": "https://registry.npmjs.org/active-x-obfuscator/-/active-x-obfuscator-0.0.1.tgz", - "integrity": "sha1-CJuJs3FF/x2ex0r2UwvlUmyuHxo=", + "integrity": "sha512-8gdEZinfLSCfAUulETDth4ZSIDPSchiPgm5PLrXQC6BANf1YFEDrPPM2MdK2zcekMROwtM667QFuYw/H6ZV06Q==", "requires": { "zeparser": "0.0.5" } @@ -348,7 +348,7 @@ "agent-base": { "version": "4.3.0", "resolved": "https://registry.npmjs.org/agent-base/-/agent-base-4.3.0.tgz", - "integrity": "sha1-gWXwHENgCbzK0LHRIvBe13Dvxu4=", + "integrity": "sha512-salcGninV0nPrwpGNn4VTXBb1SOuXQBiqbrNXoeizJsHrsL6ERFM2Ne3JUSBWRE6aeNJI2ROP/WEEIDUiDe3cg==", "requires": { "es6-promisify": "^5.0.0" } @@ -367,12 +367,12 @@ "array-flatten": { "version": "1.1.1", "resolved": "https://registry.npmjs.org/array-flatten/-/array-flatten-1.1.1.tgz", - "integrity": "sha1-ml9pkFGx5wczKPKgCJaLZOopVdI=" + "integrity": "sha512-PCVAQswWemu6UdxsDFFX/+gVeYqKAod3D3UVm91jHwynguOwAvYPhx8nNlM++NqRcK6CxxpUafjmhIdKiHibqg==" }, "arrify": { "version": "2.0.1", "resolved": "https://registry.npmjs.org/arrify/-/arrify-2.0.1.tgz", - "integrity": "sha1-yWVekzHgq81YjSp8rX6ZVvZnAfo=" + "integrity": "sha512-3duEwti880xqi4eAMN8AyR4a0ByT90zoYdLlevfrvU43vb0YZwZVfxOgxWrLXXXpyugL0hNZc9G6BiB5B3nUug==" }, "asn1": { "version": "0.2.4", @@ -385,23 +385,23 @@ "assert-plus": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/assert-plus/-/assert-plus-1.0.0.tgz", - "integrity": "sha1-8S4PPF13sLHN2RRpQuTpbB5N1SU=" + "integrity": "sha512-NfJ4UzBCcQGLDlQq7nHxH+tv3kyZ0hHQqF5BO6J7tNJeP5do1llPr8dZ8zHonfhAu0PHAdMkSo+8o0wxg9lZWw==" }, "assertion-error": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/assertion-error/-/assertion-error-1.0.0.tgz", - "integrity": "sha1-x/hUOP3UZrx8oWq5DIFRN5el0js=", + "integrity": "sha512-g/gZV+G476cnmtYI+Ko9d5khxSoCSoom/EaNmmCfwpOvBXEJ18qwFrxfP1/CsIqk2no1sAKKwxndV0tP7ROOFQ==", "dev": true }, "async": { "version": "0.9.2", "resolved": "https://registry.npmjs.org/async/-/async-0.9.2.tgz", - "integrity": "sha1-rqdNXmHB+JlhO/ZL2mbUx48v0X0=" + "integrity": "sha512-l6ToIJIotphWahxxHyzK9bnLR6kM4jJIIgLShZeqLY7iboHoGkdgFl7W2/Ivi4SkMJYGKqW8vSuk0uKUj6qsSw==" }, "async-listener": { "version": "0.6.10", "resolved": "https://registry.npmjs.org/async-listener/-/async-listener-0.6.10.tgz", - "integrity": "sha1-p8l6vlcLpgLXgic8DeYKUePhfLw=", + "integrity": "sha512-gpuo6xOyF4D5DE5WvyqZdPA3NGhiT6Qf07l7DCB0wwDEsLvDIbCr6j9S5aj5Ch96dLace5tXVzWBZkxU/c5ohw==", "requires": { "semver": "^5.3.0", "shimmer": "^1.1.0" @@ -417,12 +417,12 @@ "asynckit": { "version": "0.4.0", "resolved": "https://registry.npmjs.org/asynckit/-/asynckit-0.4.0.tgz", - "integrity": "sha1-x57Zf380y48robyXkLzDZkdLS3k=" + "integrity": "sha512-Oei9OH4tRh0YqU3GxhX79dM/mwVgvbZJaSNaRk+bshkj0S5cfHcgYakreBjrHwatXKbz+IoIdYLxrKim2MjW0Q==" }, "aws-sign2": { "version": "0.7.0", "resolved": "https://registry.npmjs.org/aws-sign2/-/aws-sign2-0.7.0.tgz", - "integrity": "sha1-tG6JCTSpWR8tL2+G1+ap8bP+dqg=" + "integrity": "sha512-08kcGqnYf/YmjoRhfxyu+CLxBjUtHLXLXX/vUfx9l2LYzG3c1m61nrpyFUZI6zeS+Li/wWMMidD9KgrqtGq3mA==" }, "aws4": { "version": "1.10.0", @@ -432,7 +432,7 @@ "axios": { "version": "0.18.1", "resolved": "https://registry.npmjs.org/axios/-/axios-0.18.1.tgz", - "integrity": "sha1-/z8N4ue10YDnV62YAA8Qgbh7zqM=", + "integrity": "sha512-0BfJq4NSfQXd+SkFdrvFbG7addhYSBA2mQwISr46pD6E5iqkWg02RAs8vyTT/j0RTnoYmeXauBuSv1qKwR179g==", "requires": { "follow-redirects": "1.5.10", "is-buffer": "^2.0.2" @@ -441,7 +441,7 @@ "balanced-match": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/balanced-match/-/balanced-match-1.0.0.tgz", - "integrity": "sha1-ibTRmasr7kneFk6gK4nORi1xt2c=" + "integrity": "sha512-9Y0g0Q8rmSt+H33DfKv7FOc3v+iRI+o1lbzt8jGcIosYW37IIW/2XVYq5NPdmaD5NQ59Nk26Kl/vZbwW9Fr8vg==" }, "base64-js": { "version": "1.3.1", @@ -451,17 +451,17 @@ "base64id": { "version": "0.1.0", "resolved": "https://registry.npmjs.org/base64id/-/base64id-0.1.0.tgz", - "integrity": "sha1-As4P3u4M709ACA4ec+g08LG/zj8=" + "integrity": "sha512-DSjtfjhAsHl9J4OJj7e4+toV2zqxJrGwVd3CLlsCp8QmicvOn7irG0Mb8brOc/nur3SdO8lIbNlY1s1ZDJdUKQ==" }, "basic-auth-connect": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/basic-auth-connect/-/basic-auth-connect-1.0.0.tgz", - "integrity": "sha1-/bC0OWLKe0BFanwrtI/hc9otISI=" + "integrity": "sha512-kiV+/DTgVro4aZifY/hwRwALBISViL5NP4aReaR2EVJEObpbUBHIkdJh/YpcoEiYt7nBodZ6U2ajZeZvSxUCCg==" }, "bcrypt-pbkdf": { "version": "1.0.2", "resolved": "https://registry.npmjs.org/bcrypt-pbkdf/-/bcrypt-pbkdf-1.0.2.tgz", - "integrity": "sha1-pDAdOJtqQ/m2f/PKEaP2Y342Dp4=", + "integrity": "sha512-qeFIXtP4MSoi6NLqO12WfqARWWuCKi2Rn/9hJLEmtB5yTNr9DqFWkJRCf2qShWzPeAMRnOgCrq0sg/KLv5ES9w==", "requires": { "tweetnacl": "^0.14.3" } @@ -469,12 +469,12 @@ "bignumber.js": { "version": "7.2.1", "resolved": "https://registry.npmjs.org/bignumber.js/-/bignumber.js-7.2.1.tgz", - "integrity": "sha1-gMBIdZ2CaACAfEv9Uh5Q7bulel8=" + "integrity": "sha512-S4XzBk5sMB+Rcb/LNcpzXr57VRTxgAvaAEDAl1AwRx27j00hT84O6OkteE7u8UB3NuaaygCRrEpqox4uDOrbdQ==" }, "bindings": { "version": "1.5.0", "resolved": "https://registry.npmjs.org/bindings/-/bindings-1.5.0.tgz", - "integrity": "sha1-EDU8npRTNLwFEabZCzj7x8nFBN8=", + "integrity": "sha512-p2q/t/mhvuOj/UeLlV6566GD/guowlr0hHxClI0W9m7MWYkL1F0hLo+0Aexs9HSPCtR1SXQ0TD3MMKrXZajbiQ==", "requires": { "file-uri-to-path": "1.0.0" } @@ -482,7 +482,7 @@ "bintrees": { "version": "1.0.1", "resolved": "https://registry.npmjs.org/bintrees/-/bintrees-1.0.1.tgz", - "integrity": "sha1-DmVcm5wkNeqraL9AJyJtK1WjRSQ=" + "integrity": "sha512-tbaUB1QpTIj4cKY8c1rvNAvEQXA+ekzHmbe4jzNfW3QWsF9GnnP/BRWyl6/qqS53heoYJ93naaFcm/jooONH8g==" }, "body-parser": { "version": "1.19.0", @@ -504,7 +504,7 @@ "brace-expansion": { "version": "1.1.11", "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-1.1.11.tgz", - "integrity": "sha1-PH/L9SnYcibz0vUrlm/1Jx60Qd0=", + "integrity": "sha512-iCuPHDFgrHX7H2vEI/5xpz07zSHB00TpugqhmYtVmMO6518mCuRMoOYFldEBl0g187ufozdaHgWKcYFb61qGiA==", "requires": { "balanced-match": "^1.0.0", "concat-map": "0.0.1" @@ -513,23 +513,23 @@ "browser-stdout": { "version": "1.3.0", "resolved": "https://registry.npmjs.org/browser-stdout/-/browser-stdout-1.3.0.tgz", - "integrity": "sha1-81HTKWnTL6XXpVZxVCY9korjvR8=", + "integrity": "sha512-7Rfk377tpSM9TWBEeHs0FlDZGoAIei2V/4MdZJoFMBFAK6BqLpxAIUepGRHGdPFgGsLb02PXovC4qddyHvQqTg==", "dev": true }, "buffer-equal-constant-time": { "version": "1.0.1", "resolved": "https://registry.npmjs.org/buffer-equal-constant-time/-/buffer-equal-constant-time-1.0.1.tgz", - "integrity": "sha1-+OcRMvf/5uAaXJaXpMbz5I1cyBk=" + "integrity": "sha512-zRpUiDwd/xk6ADqPMATG8vc9VPrkck7T07OIx0gnjmJAnHnTVXNQG3vfvWNuiZIkwu9KrKdA1iJKfsfTVxE6NA==" }, "builtin-modules": { "version": "3.1.0", "resolved": "https://registry.npmjs.org/builtin-modules/-/builtin-modules-3.1.0.tgz", - "integrity": "sha1-qtl8FRMet2tltQ7yCOdYTNdqdIQ=" + "integrity": "sha512-k0KL0aWZuBt2lrxrcASWDfwOLMnodeQjodT/1SxEQAXsHANgo6ZC/VEaSEHCXt7aSTZ4/4H5LKa+tBXmW7Vtvw==" }, "bunyan": { "version": "0.22.3", "resolved": "https://registry.npmjs.org/bunyan/-/bunyan-0.22.3.tgz", - "integrity": "sha1-ehncG0yMZF90AkGnQPIkUUfGfsI=", + "integrity": "sha512-v9dd5qmd6nJHEi7fiNo1fR2pMpE8AiB47Ap984p4iJKj+dEA69jSccmq6grFQn6pxIh0evvKpC5XO1SKfiaRoQ==", "dev": true, "requires": { "dtrace-provider": "0.2.8", @@ -544,12 +544,12 @@ "caseless": { "version": "0.12.0", "resolved": "https://registry.npmjs.org/caseless/-/caseless-0.12.0.tgz", - "integrity": "sha1-G2gcIf+EAzyCZUMJBolCDRhxUdw=" + "integrity": "sha512-4tYFyifaFfGacoiObjJegolkwSU4xQNGbVgUiNYVUxbQ2x2lUsFvY4hVgVzGiIe6WLOPqycWXA40l+PWsxthUw==" }, "chai": { "version": "1.9.2", "resolved": "https://registry.npmjs.org/chai/-/chai-1.9.2.tgz", - "integrity": "sha1-Pxog+CsLnXQ3V30k1vErGmnTtZA=", + "integrity": "sha512-olRoaitftnzWHFEAza6MXR4w+FfZrOVyV7r7U/Z8ObJefCgL8IuWkAuASJjSXrpP9wvgoL8+1dB9RbMLc2FkNg==", "dev": true, "requires": { "assertion-error": "1.0.0", @@ -564,12 +564,12 @@ "coffee-script": { "version": "1.6.0", "resolved": "https://registry.npmjs.org/coffee-script/-/coffee-script-1.6.0.tgz", - "integrity": "sha1-gIs5bhEPU9AhoZpO8fZb4OjjX6M=" + "integrity": "sha512-Tx8itEfCsQp8RbLDFt7qwjqXycAx2g6SI7//4PPUR2j6meLmNifYm6zKrNDcU1+Q/GWRhjhEZk7DaLG1TfIzGA==" }, "combined-stream": { "version": "1.0.7", "resolved": "https://registry.npmjs.org/combined-stream/-/combined-stream-1.0.7.tgz", - "integrity": "sha1-LR0kMXr7ir6V1tLAsHtXgTU52Cg=", + "integrity": "sha512-brWl9y6vOB1xYPZcpZde3N9zDByXTosAeMDo4p1wzo6UMOX4vumB+TP1RZ76sfE6Md68Q0NJSrE/gbezd4Ul+w==", "requires": { "delayed-stream": "~1.0.0" } @@ -577,12 +577,12 @@ "concat-map": { "version": "0.0.1", "resolved": "https://registry.npmjs.org/concat-map/-/concat-map-0.0.1.tgz", - "integrity": "sha1-2Klr13/Wjfd5OnMDajug1UBdR3s=" + "integrity": "sha512-/Srv4dswyQNBfohGpz9o6Yb3Gz3SrUDqBH5rTuhGR7ahtlbYKnVxw2bCFMRljaA7EXHaXZ8wsHdodFvbkhKmqg==" }, "connect-redis": { "version": "2.5.1", "resolved": "https://registry.npmjs.org/connect-redis/-/connect-redis-2.5.1.tgz", - "integrity": "sha1-6MCF227Gg7T8RXzzP5PD5+1vA/c=", + "integrity": "sha512-mnPCfN12SYcTw1yYRLejwv6WMmWpNtMkLeVIPzzAkC+u/bY8GxvMezG7wUKpMNoLmggS5xG2DWjEelggv6s5cw==", "requires": { "debug": "^1.0.4", "redis": "^0.12.1" @@ -591,7 +591,7 @@ "debug": { "version": "1.0.5", "resolved": "https://registry.npmjs.org/debug/-/debug-1.0.5.tgz", - "integrity": "sha1-9yQSF0MPmd7EwrRz6rkiKOh0wqw=", + "integrity": "sha512-SIKSrp4+XqcUaNWhwaPJbLFnvSXPsZ4xBdH2WRK0Xo++UzMC4eepYghGAVhVhOwmfq3kqowqJ5w45R3pmYZnuA==", "requires": { "ms": "2.0.0" } @@ -601,7 +601,7 @@ "console-log-level": { "version": "1.4.1", "resolved": "https://registry.npmjs.org/console-log-level/-/console-log-level-1.4.1.tgz", - "integrity": "sha1-nFprue8e9lsFq6gwKLD/iUzfYwo=" + "integrity": "sha512-VZzbIORbP+PPcN/gg3DXClTLPLg5Slwd5fL2MIc+o1qZ4BXBvWyc6QxPk6T/Mkr6IVjRpoAGf32XxP3ZWMVRcQ==" }, "content-disposition": { "version": "0.5.3", @@ -626,7 +626,7 @@ "continuation-local-storage": { "version": "3.2.1", "resolved": "https://registry.npmjs.org/continuation-local-storage/-/continuation-local-storage-3.2.1.tgz", - "integrity": "sha1-EfYT906RT+mzTJKtLSj+auHbf/s=", + "integrity": "sha512-jx44cconVqkCEEyLSKWwkvUXwO561jXMa3LPjTPsm5QR22PA0/mhe33FT4Xb5y74JDvt/Cq+5lm8S8rskLv9ZA==", "requires": { "async-listener": "^0.6.0", "emitter-listener": "^1.1.1" @@ -635,7 +635,7 @@ "cookie": { "version": "0.3.1", "resolved": "https://registry.npmjs.org/cookie/-/cookie-0.3.1.tgz", - "integrity": "sha1-5+Ch+e9DtMi6klxcWpboBtFoc7s=" + "integrity": "sha512-+IJOX0OqlHCszo2mBUq+SrEbCj6w7Kpffqx60zYbPTFaO4+yYgRjHwcZNpWvaTylDHaV7PPmBHzSecZiMhtPgw==" }, "cookie-parser": { "version": "1.4.5", @@ -654,7 +654,7 @@ "cookie-signature": { "version": "1.0.6", "resolved": "https://registry.npmjs.org/cookie-signature/-/cookie-signature-1.0.6.tgz", - "integrity": "sha1-4wOogrNCzD7oylE6eZmXNNqzriw=" + "integrity": "sha512-QADzlaHc8icV8I7vbaJXJwod9HWYp8uCqf1xa4OfNu1T7JVxQIrUgOWtHdNDtPiywmFbiS12VjotIXLrKM3orQ==" } } }, @@ -667,12 +667,12 @@ "core-util-is": { "version": "1.0.2", "resolved": "https://registry.npmjs.org/core-util-is/-/core-util-is-1.0.2.tgz", - "integrity": "sha1-tf1UIgqivFq1eqtxQMlAdUUDwac=" + "integrity": "sha512-3lqz5YjWTYnW6dlDa5TLaTCcShfar1e40rmcJVwCBJC6mWlFuj0eCHIElmG1g5kyuJ/GD+8Wn4FFCcz4gJPfaQ==" }, "dashdash": { "version": "1.14.1", "resolved": "https://registry.npmjs.org/dashdash/-/dashdash-1.14.1.tgz", - "integrity": "sha1-hTz6D3y+L+1d4gMmuN1YEDX24vA=", + "integrity": "sha512-jRFi8UDGo6j+odZiEpjazZaWqEal3w/basFjQHQEwVtZJGDpxbH1MeYluwCS8Xq5wmLJooDlMgvVarmWfGM44g==", "requires": { "assert-plus": "^1.0.0" } @@ -688,7 +688,7 @@ "deep-eql": { "version": "0.1.3", "resolved": "https://registry.npmjs.org/deep-eql/-/deep-eql-0.1.3.tgz", - "integrity": "sha1-71WKyrjeJSBs1xOQbXTlaTDrafI=", + "integrity": "sha512-6sEotTRGBFiNcqVoeHwnfopbSpi5NbH1VWJmYCVkmxMmaVTT0bUTrNaGyBwhgP4MZL012W/mkzIn3Da+iDYweg==", "dev": true, "requires": { "type-detect": "0.1.1" @@ -697,12 +697,12 @@ "delay": { "version": "4.3.0", "resolved": "https://registry.npmjs.org/delay/-/delay-4.3.0.tgz", - "integrity": "sha1-7+6/uPVFV5yzlrOnIkQ+yW0UxQ4=" + "integrity": "sha512-Lwaf3zVFDMBop1yDuFZ19F9WyGcZcGacsbdlZtWjQmM50tOcMntm1njF/Nb/Vjij3KaSvCF+sEYGKrrjObu2NA==" }, "delayed-stream": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/delayed-stream/-/delayed-stream-1.0.0.tgz", - "integrity": "sha1-3zrhmayt+31ECqrgsp4icrJOxhk=" + "integrity": "sha512-ZySD7Nf91aLB0RxL4KGrKHBXl7Eds1DAmEdcoVawXnLD7SDhpNgtuII2aAkg7a7QS41jxPSZ17p4VdGnMHk3MQ==" }, "denque": { "version": "1.4.1", @@ -712,12 +712,12 @@ "depd": { "version": "1.1.2", "resolved": "https://registry.npmjs.org/depd/-/depd-1.1.2.tgz", - "integrity": "sha1-m81S4UwJd2PnSbJ0xDRu0uVgtak=" + "integrity": "sha512-7emPTl6Dpo6JRXOXjLRxck+FlLRX5847cLKEn00PLAgc3g2hTZZgr+e4c2v6QpSmLeFP3n5yUo7ft6avBK/5jQ==" }, "destroy": { "version": "1.0.4", "resolved": "https://registry.npmjs.org/destroy/-/destroy-1.0.4.tgz", - "integrity": "sha1-l4hXRCxEdJ5CBmE+N5RiBYJqvYA=" + "integrity": "sha512-3NdhDuEXnfun/z7x9GOElY49LoqVHoGScmOKwmxhsS8N5Y+Z8KyPPDnaSzqWgYt/ji4mqwfTS34Htrk0zPIXVg==" }, "diff": { "version": "3.5.0", @@ -728,14 +728,14 @@ "dtrace-provider": { "version": "0.2.8", "resolved": "https://registry.npmjs.org/dtrace-provider/-/dtrace-provider-0.2.8.tgz", - "integrity": "sha1-4kPxkhmqlfvw2PL/sH9b1k6U/iA=", + "integrity": "sha512-wufYnYt4ISHnT9MEiRgQ3trXuolt7mICTa/ckT+KYHR667K9H82lmI8KM7zKUJ8l5I343A34wJnvL++1TJn1iA==", "dev": true, "optional": true }, "duplexify": { "version": "3.7.1", "resolved": "https://registry.npmjs.org/duplexify/-/duplexify-3.7.1.tgz", - "integrity": "sha1-Kk31MX9sz9kfhtb9JdjYoQO4gwk=", + "integrity": "sha512-07z8uv2wMyS51kKhD1KsdXJg5WQ6t93RneqRxUHnskXVtlYYkLqM0gqStQZ3pj073g687jPCHrqNfCzawLYh5g==", "requires": { "end-of-stream": "^1.0.0", "inherits": "^2.0.1", @@ -746,7 +746,7 @@ "ecc-jsbn": { "version": "0.1.2", "resolved": "https://registry.npmjs.org/ecc-jsbn/-/ecc-jsbn-0.1.2.tgz", - "integrity": "sha1-OoOpBOVDUyh4dMVkt1SThoSamMk=", + "integrity": "sha512-eh9O+hwRHNbG4BLTjEl3nw044CkGm5X6LoaCf7LPp7UU8Qrt47JYNi6nPX8xjW97TKGKm1ouctg0QSpZe9qrnw==", "requires": { "jsbn": "~0.1.0", "safer-buffer": "^2.1.0" @@ -755,7 +755,7 @@ "ecdsa-sig-formatter": { "version": "1.0.11", "resolved": "https://registry.npmjs.org/ecdsa-sig-formatter/-/ecdsa-sig-formatter-1.0.11.tgz", - "integrity": "sha1-rg8PothQRe8UqBfao86azQSJ5b8=", + "integrity": "sha512-nagl3RYrbNv6kQkeJIpt6NJZy8twLB/2vtz6yN9Z4vRKHN4/QZJIEbqohALSgwKdnksuY3k5Addp5lg8sVoVcQ==", "requires": { "safe-buffer": "^5.0.1" } @@ -763,12 +763,12 @@ "ee-first": { "version": "1.1.1", "resolved": "https://registry.npmjs.org/ee-first/-/ee-first-1.1.1.tgz", - "integrity": "sha1-WQxhFWsK4vTwJVcyoViyZrxWsh0=" + "integrity": "sha512-WMwm9LhRUo+WUaRN+vRuETqG89IgZphVSNkdFgeb6sS/E4OrDIN7t48CAewSHXc6C8lefD8KKfr5vY61brQlow==" }, "emitter-listener": { "version": "1.1.2", "resolved": "https://registry.npmjs.org/emitter-listener/-/emitter-listener-1.1.2.tgz", - "integrity": "sha1-VrFA6PaZI3Wz18ssqxzHQy2WMug=", + "integrity": "sha512-Bt1sBAGFHY9DKY+4/2cV6izcKJUf5T7/gkdmkxzX/qv9CcGH8xSwVRW5mtX03SWJtRTWSOpzCuWN9rBFYZepZQ==", "requires": { "shimmer": "^1.2.0" } @@ -776,7 +776,7 @@ "encodeurl": { "version": "1.0.2", "resolved": "https://registry.npmjs.org/encodeurl/-/encodeurl-1.0.2.tgz", - "integrity": "sha1-rT/0yG7C0CkyL1oCw6mmBslbP1k=" + "integrity": "sha512-TPJXq8JqFaVYm2CWmPvnP2Iyo4ZSM7/QKcSmuMLDObfpH5fi7RUGmd/rTDf+rut/saiDiQEeVTNgAmJEdAOx0w==" }, "end-of-stream": { "version": "1.4.4", @@ -789,17 +789,17 @@ "ent": { "version": "2.2.0", "resolved": "https://registry.npmjs.org/ent/-/ent-2.2.0.tgz", - "integrity": "sha1-6WQhkyWiHQX0RGai9obtbOX13R0=" + "integrity": "sha512-GHrMyVZQWvTIdDtpiEXdHZnFQKzeO09apj8Cbl4pKWy4i0Oprcq17usfDt5aO63swf0JOeMWjWQE/LzgSRuWpA==" }, "es6-promise": { "version": "4.2.8", "resolved": "https://registry.npmjs.org/es6-promise/-/es6-promise-4.2.8.tgz", - "integrity": "sha1-TrIVlMlyvEBVPSduUQU5FD21Pgo=" + "integrity": "sha512-HJDGx5daxeIvxdBxvG2cb9g4tEvwIk3i8+nhX0yGrYmZUzbkdg8QbDevheDB8gd0//uPj4c1EQua8Q+MViT0/w==" }, "es6-promisify": { "version": "5.0.0", "resolved": "https://registry.npmjs.org/es6-promisify/-/es6-promisify-5.0.0.tgz", - "integrity": "sha1-UQnWLz5W6pZ8S2NQWu8IKRyKUgM=", + "integrity": "sha512-C+d6UdsYDk0lMebHNR4S2NybQMMngAOnOwYBQjTOiv0MkoJMP0Myw2mgpDLBcpfCmRLxyFqYhS/CfOENq4SJhQ==", "requires": { "es6-promise": "^4.0.3" } @@ -807,23 +807,23 @@ "escape-html": { "version": "1.0.3", "resolved": "https://registry.npmjs.org/escape-html/-/escape-html-1.0.3.tgz", - "integrity": "sha1-Aljq5NPQwJdN4cFpGI7wBR0dGYg=" + "integrity": "sha512-NiSupZ4OeuGwr68lGIeym/ksIZMJodUGOSCZ/FSnTxcrekbvqrgdUxlJOMpijaKZVjAJrWrGs/6Jy8OMuyj9ow==" }, "escape-string-regexp": { "version": "1.0.5", "resolved": "https://registry.npmjs.org/escape-string-regexp/-/escape-string-regexp-1.0.5.tgz", - "integrity": "sha1-G2HAViGQqN/2rjuyzwIAyhMLhtQ=", + "integrity": "sha512-vbRorB5FUQWvla16U8R/qgaFIya2qGzwDrNmCZuYKrbdSUMG6I1ZCGQRefkRVhuOkIGVne7BQ35DSfo1qvJqFg==", "dev": true }, "etag": { "version": "1.8.1", "resolved": "https://registry.npmjs.org/etag/-/etag-1.8.1.tgz", - "integrity": "sha1-Qa4u62XvpiJorr/qg6x9eSmbCIc=" + "integrity": "sha512-aIL5Fx7mawVa300al2BnEE4iNvo1qETxLrPI/o05L7z6go7fCw1J6EQmbK4FmJ2AS7kgVF/KEZWufBfdClMcPg==" }, "event-target-shim": { "version": "5.0.1", "resolved": "https://registry.npmjs.org/event-target-shim/-/event-target-shim-5.0.1.tgz", - "integrity": "sha1-XU0+vflYPWOlMzzi3rdICrKwV4k=" + "integrity": "sha512-i/2XbnSz/uxRCU6+NdVJgKWDTM427+MqYbkQzD321DuCQJUqOuJKIA0IM2+W2xtYHdKOmZ4dR6fExsd4SXL+WQ==" }, "express": { "version": "4.17.1", @@ -870,7 +870,7 @@ "cookie-signature": { "version": "1.0.6", "resolved": "https://registry.npmjs.org/cookie-signature/-/cookie-signature-1.0.6.tgz", - "integrity": "sha1-4wOogrNCzD7oylE6eZmXNNqzriw=" + "integrity": "sha512-QADzlaHc8icV8I7vbaJXJwod9HWYp8uCqf1xa4OfNu1T7JVxQIrUgOWtHdNDtPiywmFbiS12VjotIXLrKM3orQ==" }, "safe-buffer": { "version": "5.1.2", @@ -902,7 +902,7 @@ "cookie-signature": { "version": "1.0.6", "resolved": "https://registry.npmjs.org/cookie-signature/-/cookie-signature-1.0.6.tgz", - "integrity": "sha1-4wOogrNCzD7oylE6eZmXNNqzriw=" + "integrity": "sha512-QADzlaHc8icV8I7vbaJXJwod9HWYp8uCqf1xa4OfNu1T7JVxQIrUgOWtHdNDtPiywmFbiS12VjotIXLrKM3orQ==" }, "depd": { "version": "2.0.0", @@ -919,12 +919,12 @@ "extend": { "version": "3.0.2", "resolved": "https://registry.npmjs.org/extend/-/extend-3.0.2.tgz", - "integrity": "sha1-+LETa0Bx+9jrFAr/hYsQGewpFfo=" + "integrity": "sha512-fjquC59cD7CyW6urNXK0FBufkZcoiGG80wTuPujX590cB5Ttln20E2UB4S/WARVqhXffZl2LNgS+gQdPIIim/g==" }, "extsprintf": { "version": "1.3.0", "resolved": "https://registry.npmjs.org/extsprintf/-/extsprintf-1.3.0.tgz", - "integrity": "sha1-lpGEQOMEGnpBT4xS48V06zw+HgU=" + "integrity": "sha512-11Ndz7Nv+mvAC1j0ktTa7fAb0vLyGGX+rMHNBYQviQDGU0Hw7lhctJANqbPhu9nV9/izT/IntTgZ7Im/9LJs9g==" }, "fast-deep-equal": { "version": "3.1.3", @@ -944,7 +944,7 @@ "file-uri-to-path": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/file-uri-to-path/-/file-uri-to-path-1.0.0.tgz", - "integrity": "sha1-VTp7hEb/b2hDWcRF8eN6BdrMM90=" + "integrity": "sha512-0Zt+s3L7Vf1biwWZ29aARiVYLx7iMGnEUl9x33fbB/j3jR81u/O2LbqK+Bm1CDSNDKVtJ/YjwY7TUd5SkeLQLw==" }, "finalhandler": { "version": "1.1.2", @@ -963,12 +963,12 @@ "findit2": { "version": "2.2.3", "resolved": "https://registry.npmjs.org/findit2/-/findit2-2.2.3.tgz", - "integrity": "sha1-WKRmaX34piBc39vzlVNri9d3pfY=" + "integrity": "sha512-lg/Moejf4qXovVutL0Lz4IsaPoNYMuxt4PA0nGqFxnJ1CTTGGlEO2wKgoDpwknhvZ8k4Q2F+eesgkLbG2Mxfog==" }, "follow-redirects": { "version": "1.5.10", "resolved": "https://registry.npmjs.org/follow-redirects/-/follow-redirects-1.5.10.tgz", - "integrity": "sha1-e3qfmuov3/NnhqlP9kPtB/T/Xio=", + "integrity": "sha512-0V5l4Cizzvqt5D44aTXbFZz+FtyXV1vrDN6qrelxtfYQKW0KO0W2T/hkE8xvGa/540LkZlkaUjO4ailYTFtHVQ==", "requires": { "debug": "=3.1.0" }, @@ -976,7 +976,7 @@ "debug": { "version": "3.1.0", "resolved": "https://registry.npmjs.org/debug/-/debug-3.1.0.tgz", - "integrity": "sha1-W7WgZyYotkFJVmuhaBnmFRjGcmE=", + "integrity": "sha512-OX8XqP7/1a9cqkxYw2yXss15f26NKWBpDXQd0/uK/KPqdQhxbPa994hnzjcE2VqQpDslf55723cKPUOGSmMY3g==", "requires": { "ms": "2.0.0" } @@ -986,7 +986,7 @@ "forever-agent": { "version": "0.6.1", "resolved": "https://registry.npmjs.org/forever-agent/-/forever-agent-0.6.1.tgz", - "integrity": "sha1-+8cfDEGt6zf5bFd60e1C2P2sypE=" + "integrity": "sha512-j0KLYPhm6zeac4lz3oJ3o65qvgQCcPubiyotZrXqEaG4hNagNYO8qdlUrX5vwqv9ohqeT/Z3j6+yW067yWWdUw==" }, "form-data": { "version": "2.3.3", @@ -1001,7 +1001,7 @@ "formatio": { "version": "1.2.0", "resolved": "https://registry.npmjs.org/formatio/-/formatio-1.2.0.tgz", - "integrity": "sha1-87IWfZBoxGmKjVH092CjmlTYGOs=", + "integrity": "sha512-YAF05v8+XCxAyHOdiiAmHdgCVPrWO8X744fYIPtBciIorh5LndWfi1gjeJ16sTbJhzek9kd+j3YByhohtz5Wmg==", "dev": true, "requires": { "samsam": "1.x" @@ -1010,23 +1010,23 @@ "forwarded": { "version": "0.1.2", "resolved": "https://registry.npmjs.org/forwarded/-/forwarded-0.1.2.tgz", - "integrity": "sha1-mMI9qxF1ZXuMBXPozszZGw/xjIQ=" + "integrity": "sha512-Ua9xNhH0b8pwE3yRbFfXJvfdWF0UHNCdeyb2sbi9Ul/M+r3PTdrz7Cv4SCfZRMjmzEM9PhraqfZFbGTIg3OMyA==" }, "fresh": { "version": "0.5.2", "resolved": "https://registry.npmjs.org/fresh/-/fresh-0.5.2.tgz", - "integrity": "sha1-PYyt2Q2XZWn6g1qx+OSyOhBWBac=" + "integrity": "sha512-zJ2mQYM18rEFOudeV4GShTGIQ7RbzA7ozbU9I/XBpm7kqgMywgmylMwXHxZJmkVoYkna9d2pVXVXPdYTP9ej8Q==" }, "fs.realpath": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/fs.realpath/-/fs.realpath-1.0.0.tgz", - "integrity": "sha1-FQStJSMVjKpA20onh8sBQRmU6k8=", + "integrity": "sha512-OO0pH2lK6a0hZnAdau5ItzHPI6pUlvI7jMVnxUQRtw4owF2wk8lOSabtGDCTP4Ggrg2MbGnWO9X8K1t4+fGMDw==", "dev": true }, "gaxios": { "version": "1.8.4", "resolved": "https://registry.npmjs.org/gaxios/-/gaxios-1.8.4.tgz", - "integrity": "sha1-4Iw0/pPAqbZ6Ure556ZOZDX5ozk=", + "integrity": "sha512-BoENMnu1Gav18HcpV9IleMPZ9exM+AvUjrAOV4Mzs/vfz2Lu/ABv451iEXByKiMPn2M140uul1txXCg83sAENw==", "requires": { "abort-controller": "^3.0.0", "extend": "^3.0.2", @@ -1037,7 +1037,7 @@ "gcp-metadata": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/gcp-metadata/-/gcp-metadata-1.0.0.tgz", - "integrity": "sha1-UhJEAin6CZ/C98KlzcuVV16bLKY=", + "integrity": "sha512-Q6HrgfrCQeEircnNP3rCcEgiDv7eF9+1B+1MMgpE190+/+0mjQR8PxeOaRgxZWmdDAF9EIryHB9g1moPiw1SbQ==", "requires": { "gaxios": "^1.0.2", "json-bigint": "^0.3.0" @@ -1046,7 +1046,7 @@ "getpass": { "version": "0.1.7", "resolved": "https://registry.npmjs.org/getpass/-/getpass-0.1.7.tgz", - "integrity": "sha1-Xv+OPmhNVprkyysSgmBOi6YhSfo=", + "integrity": "sha512-0fzj9JxOLfJ+XGLhR8ze3unN0KZCgZwiSSDz168VERjK8Wl8kVSdcu2kspd4s4wtAa1y/qrVRiAA0WclVsu0ng==", "requires": { "assert-plus": "^1.0.0" } @@ -1054,7 +1054,7 @@ "glob": { "version": "6.0.4", "resolved": "https://registry.npmjs.org/glob/-/glob-6.0.4.tgz", - "integrity": "sha1-DwiGD2oVUSey+t1PnOJLGqtuTSI=", + "integrity": "sha512-MKZeRNyYZAVVVG1oZeLaWie1uweH40m9AZwIwxyPbTSX4hHrVYSzLg0Ro5Z5R7XKkIX+Cc6oD1rqeDJnwsB8/A==", "optional": true, "requires": { "inflight": "^1.0.4", @@ -1067,7 +1067,7 @@ "google-auth-library": { "version": "3.1.2", "resolved": "https://registry.npmjs.org/google-auth-library/-/google-auth-library-3.1.2.tgz", - "integrity": "sha1-/y+IzVzSEYpXvT1a08CTyIN/w1A=", + "integrity": "sha512-cDQMzTotwyWMrg5jRO7q0A4TL/3GWBgO7I7q5xGKNiiFf9SmGY/OJ1YsLMgI2MVHHsEGyrqYnbnmV1AE+Z6DnQ==", "requires": { "base64-js": "^1.3.0", "fast-text-encoding": "^1.0.0", @@ -1090,7 +1090,7 @@ "google-p12-pem": { "version": "1.0.4", "resolved": "https://registry.npmjs.org/google-p12-pem/-/google-p12-pem-1.0.4.tgz", - "integrity": "sha1-t3+4M6Lrn388aJ4uVPCVJ293dgU=", + "integrity": "sha512-SwLAUJqUfTB2iS+wFfSS/G9p7bt4eWcc2LyfvmUXe7cWp6p3mpxDo6LLI29MXdU6wvPcQ/up298X7GMC5ylAlA==", "requires": { "node-forge": "^0.8.0", "pify": "^4.0.0" @@ -1099,7 +1099,7 @@ "gtoken": { "version": "2.3.3", "resolved": "https://registry.npmjs.org/gtoken/-/gtoken-2.3.3.tgz", - "integrity": "sha1-in/hVcXODEtxyIbPsoKpBg2UpkE=", + "integrity": "sha512-EaB49bu/TCoNeQjhCYKI/CurooBKkGxIqFHsWABW0b25fobBYVTMe84A8EBVVZhl8emiUdNypil9huMOTmyAnw==", "requires": { "gaxios": "^1.0.4", "google-p12-pem": "^1.0.0", @@ -1111,14 +1111,14 @@ "mime": { "version": "2.4.4", "resolved": "https://registry.npmjs.org/mime/-/mime-2.4.4.tgz", - "integrity": "sha1-vXuRE1/GsBzePpuuM9ZZtj2IV+U=" + "integrity": "sha512-LRxmNwziLPT828z+4YkNzloCFC2YM4wrB99k+AV5ZbEyfGNWfG8SO1FUXLmLDBSo89NrJZ4DIWeLjy1CHGhMGA==" } } }, "har-schema": { "version": "2.0.0", "resolved": "https://registry.npmjs.org/har-schema/-/har-schema-2.0.0.tgz", - "integrity": "sha1-qUwiJOvKwEeCoNkDVSHyRzW37JI=" + "integrity": "sha512-Oqluz6zhGX8cyRaTQlFMPw80bSJVG2x/cFb8ZPhUILGgHka9SsokCCOQgpveePerqidZOrT14ipqfJb7ILcW5Q==" }, "har-validator": { "version": "5.1.3", @@ -1132,13 +1132,13 @@ "he": { "version": "1.1.1", "resolved": "https://registry.npmjs.org/he/-/he-1.1.1.tgz", - "integrity": "sha1-k0EP0hsAlzUVH4howvJx80J+I/0=", + "integrity": "sha512-z/GDPjlRMNOa2XJiB4em8wJpuuBfrFOlYKTZxtpkdr1uPdibHI8rYA3MY0KDObpVyaes0e/aunid/t88ZI2EKA==", "dev": true }, "hex2dec": { "version": "1.1.2", "resolved": "https://registry.npmjs.org/hex2dec/-/hex2dec-1.1.2.tgz", - "integrity": "sha1-jhzkvvNqdPfVcjw/swkMKGAHczg=" + "integrity": "sha512-Yu+q/XWr2fFQ11tHxPq4p4EiNkb2y+lAacJNhAdRXVfRIcDH6gi7htWFnnlIzvqHMHoWeIsfXlNAjZInpAOJDA==" }, "http-errors": { "version": "1.7.2", @@ -1155,7 +1155,7 @@ "http-signature": { "version": "1.2.0", "resolved": "https://registry.npmjs.org/http-signature/-/http-signature-1.2.0.tgz", - "integrity": "sha1-muzZJRFHcvPZW2WmCruPfBj7rOE=", + "integrity": "sha512-CAbnr6Rz4CYQkLYUtSNXxQPUH2gK8f3iWexVlsnMeD+GjlsQ0Xsy1cOX+mN3dtxYomRy21CiOzU8Uhw6OwncEQ==", "requires": { "assert-plus": "^1.0.0", "jsprim": "^1.2.2", @@ -1197,7 +1197,7 @@ "inflight": { "version": "1.0.6", "resolved": "https://registry.npmjs.org/inflight/-/inflight-1.0.6.tgz", - "integrity": "sha1-Sb1jMdfQLQwJvJEKEHW6gWW1bfk=", + "integrity": "sha512-k92I/b08q4wvFscXCLvqfsHCrjrF7yiXsQuIVvVE7N82W3+aqpzuUdBbfhWcy/FZR3/4IgflMgKLOsvPDrGCJA==", "requires": { "once": "^1.3.0", "wrappy": "1" @@ -1206,7 +1206,7 @@ "inherits": { "version": "2.0.3", "resolved": "https://registry.npmjs.org/inherits/-/inherits-2.0.3.tgz", - "integrity": "sha1-Yzwsg+PaQqUC9SRmAiSA9CCCYd4=" + "integrity": "sha512-x00IRNXNy63jwGkJmzPigoySHbaqpNuzKbBOmzK+g2OdZpQ9w+sxCN+VSB3ja7IAge2OP2qpfxTjeNcyjmW1uw==" }, "ioredis": { "version": "4.17.3", @@ -1247,7 +1247,7 @@ "is": { "version": "3.3.0", "resolved": "https://registry.npmjs.org/is/-/is-3.3.0.tgz", - "integrity": "sha1-Yc/23TxBk9uUo9YlggcrROVkXXk=" + "integrity": "sha512-nW24QBoPcFGGHJGUwnfpI7Yc5CdqWNdsyHQszVE/z2pKHXzh7FZ5GWhJqSyaQ9wMkQnsTx+kAI8bHlCX4tKdbg==" }, "is-buffer": { "version": "2.0.4", @@ -1257,27 +1257,27 @@ "is-typedarray": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/is-typedarray/-/is-typedarray-1.0.0.tgz", - "integrity": "sha1-5HnICFjfDBsR3dppQPlgEfzaSpo=" + "integrity": "sha512-cyA56iCMHAh5CdzjJIa4aohJyeO1YbwLi3Jc35MmRU6poroFjIGZzUzupGiRPOjgHg9TLu43xbpwXk523fMxKA==" }, "isarray": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/isarray/-/isarray-1.0.0.tgz", - "integrity": "sha1-u5NdSFgsuhaMBoNJV6VKPgcSTxE=" + "integrity": "sha512-VLghIWNM6ELQzo7zwmcg0NmTVyWKYjvIeM83yjp0wRDTmUnrM678fQbcKBo6n2CJEF0szoG//ytg+TKla89ALQ==" }, "isstream": { "version": "0.1.2", "resolved": "https://registry.npmjs.org/isstream/-/isstream-0.1.2.tgz", - "integrity": "sha1-R+Y/evVa+m+S4VAOaQ64uFKcCZo=" + "integrity": "sha512-Yljz7ffyPbrLpLngrMtZ7NduUgVvi6wG9RJ9IUcyCd59YQ911PBJphODUcbOVbqYfxe1wuYf/LJ8PauMRwsM/g==" }, "jsbn": { "version": "0.1.1", "resolved": "https://registry.npmjs.org/jsbn/-/jsbn-0.1.1.tgz", - "integrity": "sha1-peZUwuWi3rXyAdls77yoDA7y9RM=" + "integrity": "sha512-UVU9dibq2JcFWxQPA6KCqj5O42VOmAY3zQUfEKxU0KpTGXwNoCjkX1e13eHNvw/xPynt6pU0rZ1htjWTNTSXsg==" }, "json-bigint": { "version": "0.3.0", "resolved": "https://registry.npmjs.org/json-bigint/-/json-bigint-0.3.0.tgz", - "integrity": "sha1-DM2RLEuCcNBfBW+9E4FLU9OCWx4=", + "integrity": "sha512-u+c/u/F+JNPUekHCFyGVycRPyh9UHD5iUhSyIAn10kxbDTJxijwAbT6XHaONEOXuGGfmWUSroheXgHcml4gLgg==", "requires": { "bignumber.js": "^7.0.0" } @@ -1285,7 +1285,7 @@ "json-schema": { "version": "0.2.3", "resolved": "https://registry.npmjs.org/json-schema/-/json-schema-0.2.3.tgz", - "integrity": "sha1-tIDIkuWaLwWVTOcnvT8qTogvnhM=" + "integrity": "sha512-a3xHnILGMtk+hDOqNwHzF6e2fNbiMrXZvxKQiEv2MlgQP+pjIOzqAmKYD2mDpXYE/44M7g+n9p2bKkYWDUcXCQ==" }, "json-schema-traverse": { "version": "0.4.1", @@ -1295,12 +1295,12 @@ "json-stringify-safe": { "version": "5.0.1", "resolved": "https://registry.npmjs.org/json-stringify-safe/-/json-stringify-safe-5.0.1.tgz", - "integrity": "sha1-Epai1Y/UXxmg9s4B1lcB4sc1tus=" + "integrity": "sha512-ZClg6AaYvamvYEE82d3Iyd3vSSIjQ+odgjaTzRuO3s7toCdFKczob2i0zCh7JE8kWn17yvAWhUVxvqGwUalsRA==" }, "jsprim": { "version": "1.4.1", "resolved": "https://registry.npmjs.org/jsprim/-/jsprim-1.4.1.tgz", - "integrity": "sha1-MT5mvB5cwG5Di8G3SZwuXFastqI=", + "integrity": "sha512-4Dj8Rf+fQ+/Pn7C5qeEX02op1WfOss3PKTE9Nsop3Dx+6UPxlm1dr/og7o2cRa5hNN07CACr4NFzRLtj/rjWog==", "requires": { "assert-plus": "1.0.0", "extsprintf": "1.3.0", @@ -1311,7 +1311,7 @@ "jwa": { "version": "1.4.1", "resolved": "https://registry.npmjs.org/jwa/-/jwa-1.4.1.tgz", - "integrity": "sha1-dDwymFy56YZVUw1TZBtmyGRbA5o=", + "integrity": "sha512-qiLX/xhEEFKUAJ6FiBMbes3w9ATzyk5W7Hvzpa/SLYdxNtng+gcurvrI7TbACjIXlsJyr05/S1oUhZrc63evQA==", "requires": { "buffer-equal-constant-time": "1.0.1", "ecdsa-sig-formatter": "1.0.11", @@ -1321,7 +1321,7 @@ "jws": { "version": "3.2.2", "resolved": "https://registry.npmjs.org/jws/-/jws-3.2.2.tgz", - "integrity": "sha1-ABCZ82OUaMlBQADpmZX6UvtHgwQ=", + "integrity": "sha512-YHlZCB6lMTllWDtSPHz/ZXTsi8S00usEV6v1tjq8tOUZzw7DpSDWVXjXDre6ed1w/pd495ODpHZYSdkRTsa0HA==", "requires": { "jwa": "^1.4.1", "safe-buffer": "^5.0.1" @@ -1335,22 +1335,22 @@ "lodash.defaults": { "version": "4.2.0", "resolved": "https://registry.npmjs.org/lodash.defaults/-/lodash.defaults-4.2.0.tgz", - "integrity": "sha1-0JF4cW/+pN3p5ft7N/bwgCJ0WAw=" + "integrity": "sha512-qjxPLHd3r5DnsdGacqOMU6pb/avJzdh9tFX2ymgoZE27BmjXrNy/y4LoaiTeAb+O3gL8AfpJGtqfX/ae2leYYQ==" }, "lodash.flatten": { "version": "4.4.0", "resolved": "https://registry.npmjs.org/lodash.flatten/-/lodash.flatten-4.4.0.tgz", - "integrity": "sha1-8xwiIlqWMtK7+OSt2+8kCqdlph8=" + "integrity": "sha512-C5N2Z3DgnnKr0LOpv/hKCgKdb7ZZwafIrsesve6lmzvZIRZRGaZ/l6Q8+2W7NaT+ZwO3fFlSCzCzrDCFdJfZ4g==" }, "lodash.pickby": { "version": "4.6.0", "resolved": "https://registry.npmjs.org/lodash.pickby/-/lodash.pickby-4.6.0.tgz", - "integrity": "sha1-feoh2MGNdwOifHBMFdO4SmfjOv8=" + "integrity": "sha512-AZV+GsS/6ckvPOVQPXSiFFacKvKB4kOQu6ynt9wz0F3LO4R9Ij4K1ddYsIytDpSgLz88JHd9P+oaLeej5/Sl7Q==" }, "logger-sharelatex": { "version": "1.7.0", "resolved": "https://registry.npmjs.org/logger-sharelatex/-/logger-sharelatex-1.7.0.tgz", - "integrity": "sha1-XuMje84im1rITZ7SLoXa6eI3/HQ=", + "integrity": "sha512-9sxDGPSphOMDqUqGpOu/KxFAVcpydKggWv60g9D7++FDCxGkhLLn0kmBkDdgB00d1PadgX1CBMWKzIBpptDU/Q==", "requires": { "bunyan": "1.8.12", "raven": "1.1.3", @@ -1360,7 +1360,7 @@ "bunyan": { "version": "1.8.12", "resolved": "https://registry.npmjs.org/bunyan/-/bunyan-1.8.12.tgz", - "integrity": "sha1-8VDw9nSKvdcq6uhPBEA74u8RN5c=", + "integrity": "sha512-dmDUbGHeGcvCDLRFOscZkwx1ZO/aFz3bJOCi5nCgzdhFGPxwK+y5AcDBnqagNGlJZ7lje/l6JUEz9mQcutttdg==", "requires": { "dtrace-provider": "~0.8", "moment": "^2.10.6", @@ -1371,7 +1371,7 @@ "dtrace-provider": { "version": "0.8.7", "resolved": "https://registry.npmjs.org/dtrace-provider/-/dtrace-provider-0.8.7.tgz", - "integrity": "sha1-3JObTT4GIM/gwc2APQ0tftBP/QQ=", + "integrity": "sha512-V+HIGbAdxCIxddHNDwzXi6cx8Cz5RRlQOVcsryHfsyVVebpBEnDwHSgqxpgKzqeU/6/0DWqRLAGUwkbg2ecN1Q==", "optional": true, "requires": { "nan": "^2.10.0" @@ -1437,18 +1437,18 @@ "lolex": { "version": "1.6.0", "resolved": "https://registry.npmjs.org/lolex/-/lolex-1.6.0.tgz", - "integrity": "sha1-OpoCg0UqR9dDnnJzG54H1zhuSfY=", + "integrity": "sha512-/bpxDL56TG5LS5zoXxKqA6Ro5tkOS5M8cm/7yQcwLIKIcM2HR5fjjNCaIhJNv96SEk4hNGSafYMZK42Xv5fihQ==", "dev": true }, "long": { "version": "4.0.0", "resolved": "https://registry.npmjs.org/long/-/long-4.0.0.tgz", - "integrity": "sha1-mntxz7fTYaGU6lVSQckvdGjVvyg=" + "integrity": "sha512-XsP+KhQif4bjX1kbuSiySJFNAehNxgLb6hPRGJ9QsUr8ajHkuXGdrHmFUTUUXhDwVX2R5bY4JNZEwbUiMhV+MA==" }, "lru-cache": { "version": "5.1.1", "resolved": "https://registry.npmjs.org/lru-cache/-/lru-cache-5.1.1.tgz", - "integrity": "sha1-HaJ+ZxAnGUdpXa9oSOhH8B2EuSA=", + "integrity": "sha512-KpNARQA3Iwv+jTA0utUVVbrh+Jlrr1Fv0e56GGzAFOXN7dk/FviaDW8LHmK52DlcH4WP2n6gI8vN1aesBFgo9w==", "requires": { "yallist": "^3.0.2" } @@ -1456,12 +1456,12 @@ "lsmod": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/lsmod/-/lsmod-1.0.0.tgz", - "integrity": "sha1-mgD3bco26yP6BTUK/htYXUKZ5ks=" + "integrity": "sha512-Y+6V75r+mGWzWEPr9h6PFmStielICu5JBHLUg18jCsD2VFmEfgHbq/EgnY4inElsUD9eKL9id1qp34w46rSIKQ==" }, "lynx": { "version": "0.1.1", "resolved": "https://registry.npmjs.org/lynx/-/lynx-0.1.1.tgz", - "integrity": "sha1-Mxjc7xaQi4KG6Bisz9sxzXQkj50=", + "integrity": "sha512-JI52N0NwK2b/Md0TFPdPtUBI46kjyJXF7+q08l2yvQ56q6QA8s7ZjZQQRoxFpS2jDXNf/B0p8ID+OIKcTsZwzw==", "requires": { "mersenne": "~0.0.3", "statsd-parser": "~0.0.4" @@ -1470,22 +1470,22 @@ "media-typer": { "version": "0.3.0", "resolved": "https://registry.npmjs.org/media-typer/-/media-typer-0.3.0.tgz", - "integrity": "sha1-hxDXrwqmJvj/+hzgAWhUUmMlV0g=" + "integrity": "sha512-dq+qelQ9akHpcOl/gUVRTxVIOkAJ1wR3QAvb4RsVjS8oVoFjDGTc679wJYmUmknUF5HwMLOgb5O+a3KxfWapPQ==" }, "merge-descriptors": { "version": "1.0.1", "resolved": "https://registry.npmjs.org/merge-descriptors/-/merge-descriptors-1.0.1.tgz", - "integrity": "sha1-sAqqVW3YtEVoFQ7J0blT8/kMu2E=" + "integrity": "sha512-cCi6g3/Zr1iqQi6ySbseM1Xvooa98N0w31jzUYrXPX2xqObmFGHJ0tQ5u74H3mVh7wLouTseZyYIq39g8cNp1w==" }, "mersenne": { "version": "0.0.4", "resolved": "https://registry.npmjs.org/mersenne/-/mersenne-0.0.4.tgz", - "integrity": "sha1-QB/ex+whzbngPNPTAhOY2iGycIU=" + "integrity": "sha512-XoSUL+nF8hMTKGQxUs8r3Btdsf1yuKKBdCCGbh3YXgCXuVKishpZv1CNc385w9s8t4Ynwc5h61BwW/FCVulkbg==" }, "methods": { "version": "1.1.2", "resolved": "https://registry.npmjs.org/methods/-/methods-1.1.2.tgz", - "integrity": "sha1-VSmk1nZUE07cxSZmVoNbD4Ua/O4=" + "integrity": "sha512-iclAHeNqNm68zFtnZ0e+1L2yUIdvzNoauKU4WBA3VvH/vPFieF7qfRlwUZU+DA9P9bPXIS90ulxoUoCH23sV2w==" }, "metrics-sharelatex": { "version": "2.6.2", @@ -1505,7 +1505,7 @@ "underscore": { "version": "1.6.0", "resolved": "https://registry.npmjs.org/underscore/-/underscore-1.6.0.tgz", - "integrity": "sha1-izixDKze9jM3uLJOT/htRa6lKag=" + "integrity": "sha512-z4o1fvKUojIWh9XuaVLUDdf86RQiq13AC1dmHbTpoyuu+bquHms76v16CjycCbec87J7z0k//SiQVk0sMdFmpQ==" } } }, @@ -1517,12 +1517,12 @@ "mime-db": { "version": "1.33.0", "resolved": "https://registry.npmjs.org/mime-db/-/mime-db-1.33.0.tgz", - "integrity": "sha1-o0kgUKXLm2NFBUHjnZeI0icng9s=" + "integrity": "sha512-BHJ/EKruNIqJf/QahvxwQZXKygOQ256myeN/Ew+THcAa5q+PjyTTMMeNQC4DZw5AwfvelsUrA6B67NKMqXDbzQ==" }, "mime-types": { "version": "2.1.18", "resolved": "https://registry.npmjs.org/mime-types/-/mime-types-2.1.18.tgz", - "integrity": "sha1-bzI/YKg9ERRvgx/xH9ZuL+VQO7g=", + "integrity": "sha512-lc/aahn+t4/SWV/qcmumYjymLsWfN3ELhpmVuUFjgsORruuZPVSwAQryq+HHGvO/SI2KVX26bx+En+zhM8g8hQ==", "requires": { "mime-db": "~1.33.0" } @@ -1530,7 +1530,7 @@ "minimatch": { "version": "3.0.4", "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-3.0.4.tgz", - "integrity": "sha1-UWbihkV/AzBgZL5Ul+jbsMPTIIM=", + "integrity": "sha512-yJHVQEhyqPLUTgt9B83PXu6W3rx4MvvHvSUvToogpwoGDOUQ+yDrR0HRot+yOCdCO7u4hX3pWft6kWBBcqh0UA==", "requires": { "brace-expansion": "^1.1.7" } @@ -1538,12 +1538,12 @@ "minimist": { "version": "0.0.8", "resolved": "https://registry.npmjs.org/minimist/-/minimist-0.0.8.tgz", - "integrity": "sha1-hX/Kv8M5fSYluCKCYuhqp6ARsF0=" + "integrity": "sha512-miQKw5Hv4NS1Psg2517mV4e4dYNaO3++hjAvLOAzKqZ61rH8NS1SK+vbfBWZ5PY/Me/bEWhUwqMghEW5Fb9T7Q==" }, "mkdirp": { "version": "0.5.1", "resolved": "https://registry.npmjs.org/mkdirp/-/mkdirp-0.5.1.tgz", - "integrity": "sha1-MAV0OOrGz3+MR2fzhkjWaX11yQM=", + "integrity": "sha512-SknJC52obPfGQPnjIkXbmA6+5H15E+fR+E4iR2oQ3zzCLbd7/ONua69R/Gw7AgkTLsRG+r5fzksYwWe1AgTyWA==", "requires": { "minimist": "0.0.8" } @@ -1551,7 +1551,7 @@ "mocha": { "version": "4.1.0", "resolved": "https://registry.npmjs.org/mocha/-/mocha-4.1.0.tgz", - "integrity": "sha1-fYbPvPNcuCnidUwy4XNV7AUzh5Q=", + "integrity": "sha512-0RVnjg1HJsXY2YFDoTNzcc1NKhYuXKRrBAG2gDygmJJA136Cs2QlRliZG1mA0ap7cuaT30mw16luAeln+4RiNA==", "dev": true, "requires": { "browser-stdout": "1.3.0", @@ -1569,13 +1569,13 @@ "commander": { "version": "2.11.0", "resolved": "https://registry.npmjs.org/commander/-/commander-2.11.0.tgz", - "integrity": "sha1-FXFS/R56bI2YpbcVzzdt+SgARWM=", + "integrity": "sha512-b0553uYA5YAEGgyYIGYROzKQ7X5RAqedkfjiZxwi0kL1g3bOaBNNZfYkzt/CL0umgD5wc9Jec2FbB98CjkMRvQ==", "dev": true }, "debug": { "version": "3.1.0", "resolved": "https://registry.npmjs.org/debug/-/debug-3.1.0.tgz", - "integrity": "sha1-W7WgZyYotkFJVmuhaBnmFRjGcmE=", + "integrity": "sha512-OX8XqP7/1a9cqkxYw2yXss15f26NKWBpDXQd0/uK/KPqdQhxbPa994hnzjcE2VqQpDslf55723cKPUOGSmMY3g==", "dev": true, "requires": { "ms": "2.0.0" @@ -1584,13 +1584,13 @@ "diff": { "version": "3.3.1", "resolved": "https://registry.npmjs.org/diff/-/diff-3.3.1.tgz", - "integrity": "sha1-qoVnpu7QPFMfyJ0/cRzQ5SWd7HU=", + "integrity": "sha512-MKPHZDMB0o6yHyDryUOScqZibp914ksXwAMYMTHj6KO8UeKsRYNJD3oNCKjTqZon+V488P7N/HzXF8t7ZR95ww==", "dev": true }, "glob": { "version": "7.1.2", "resolved": "https://registry.npmjs.org/glob/-/glob-7.1.2.tgz", - "integrity": "sha1-wZyd+aAocC1nhhI4SmVSQExjbRU=", + "integrity": "sha512-MJTUg1kjuLeQCJ+ccE4Vpa6kKVXkPYJ2mOCQyUuKLcLQsdrMCpBPUi8qVE6+YuaJkozeA9NusTAw3hLr8Xe5EQ==", "dev": true, "requires": { "fs.realpath": "^1.0.0", @@ -1604,19 +1604,19 @@ "growl": { "version": "1.10.3", "resolved": "https://registry.npmjs.org/growl/-/growl-1.10.3.tgz", - "integrity": "sha1-GSa6kM8+3+KttJJ/WIC8IsZseQ8=", + "integrity": "sha512-hKlsbA5Vu3xsh1Cg3J7jSmX/WaW6A5oBeqzM88oNbCRQFz+zUaXm6yxS4RVytp1scBoJzSYl4YAEOQIt6O8V1Q==", "dev": true }, "has-flag": { "version": "2.0.0", "resolved": "https://registry.npmjs.org/has-flag/-/has-flag-2.0.0.tgz", - "integrity": "sha1-6CB68cx7MNRGzHC3NLXovhj4jVE=", + "integrity": "sha512-P+1n3MnwjR/Epg9BBo1KT8qbye2g2Ou4sFumihwt6I4tsUX7jnLcX4BTOSKg/B1ZrIYMN9FcEnG4x5a7NB8Eng==", "dev": true }, "supports-color": { "version": "4.4.0", "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-4.4.0.tgz", - "integrity": "sha1-iD992rwWUUKyphQn8zUt7RldGj4=", + "integrity": "sha512-rKC3+DyXWgK0ZLKwmRsrkyHVZAjNkfzeehuFWdGGcqGDTZFH73+RH6S/RDAAxl9GusSjZSUWYLmT9N5pzXFOXQ==", "dev": true, "requires": { "has-flag": "^2.0.0" @@ -1627,23 +1627,23 @@ "module-details-from-path": { "version": "1.0.3", "resolved": "https://registry.npmjs.org/module-details-from-path/-/module-details-from-path-1.0.3.tgz", - "integrity": "sha1-EUyUlnPiqKNenTV4hSeqN7Z52is=" + "integrity": "sha512-ySViT69/76t8VhE1xXHK6Ch4NcDd26gx0MzKXLO+F7NOtnqH68d9zF94nT8ZWSxXh8ELOERsnJO/sWt1xZYw5A==" }, "moment": { "version": "2.24.0", "resolved": "https://registry.npmjs.org/moment/-/moment-2.24.0.tgz", - "integrity": "sha1-DQVdU/UFKqZTyfbraLtdEr9cK1s=", + "integrity": "sha512-bV7f+6l2QigeBBZSM/6yTNq4P2fNpSWj/0e7jQcy87A8e7o2nAfP/34/2ky5Vw4B9S446EtIhodAzkFCcR4dQg==", "optional": true }, "ms": { "version": "2.0.0", "resolved": "https://registry.npmjs.org/ms/-/ms-2.0.0.tgz", - "integrity": "sha1-VgiurfwAvmwpAd9fmGF4jeDVl8g=" + "integrity": "sha512-Tpp60P6IUJDTuOq/5Z8cdskzJujfwqfOTkrwIwj7IRISpnkJnT6SyJ4PCPnGMoFjC9ddhal5KVIYtAt97ix05A==" }, "mv": { "version": "2.1.1", "resolved": "https://registry.npmjs.org/mv/-/mv-2.1.1.tgz", - "integrity": "sha1-rmzg1vbV4KT32JN5jQPB6pVZtqI=", + "integrity": "sha512-at/ZndSy3xEGJ8i0ygALh8ru9qy7gWW1cmkaqBN29JmMlIvM//MEO9y1sk/avxuwnPcfhkejkLsuPxH81BrkSg==", "optional": true, "requires": { "mkdirp": "~0.5.1", @@ -1654,18 +1654,18 @@ "nan": { "version": "2.12.1", "resolved": "https://registry.npmjs.org/nan/-/nan-2.12.1.tgz", - "integrity": "sha1-exqhk+mqhgV+PHu9CsRI53CSVVI=" + "integrity": "sha512-JY7V6lRkStKcKTvHO5NVSQRv+RV+FIL5pvDoLiAtSL9pKlC5x9PKQcZDsq7m4FO4d57mkhC6Z+QhAh3Jdk5JFw==" }, "native-promise-only": { "version": "0.8.1", "resolved": "https://registry.npmjs.org/native-promise-only/-/native-promise-only-0.8.1.tgz", - "integrity": "sha1-IKMYwwy0X3H+et+/eyHJnBRy7xE=", + "integrity": "sha512-zkVhZUA3y8mbz652WrL5x0fB0ehrBkulWT3TomAQ9iDtyXZvzKeEA6GPxAItBYeNYl5yngKRX612qHOhvMkDeg==", "dev": true }, "ncp": { "version": "2.0.0", "resolved": "https://registry.npmjs.org/ncp/-/ncp-2.0.0.tgz", - "integrity": "sha1-GVoh1sRuNh0vsSgbo4uR6d9727M=", + "integrity": "sha512-zIdGUrPRFTUELUvr3Gmc7KZ2Sw/h1PiVM0Af/oHB6zgnV1ikqSfRk+TOufi79aHYCW3NiOXmr1BP5nWbzojLaA==", "optional": true }, "negotiator": { @@ -1676,7 +1676,7 @@ "node-fetch": { "version": "2.6.0", "resolved": "https://registry.npmjs.org/node-fetch/-/node-fetch-2.6.0.tgz", - "integrity": "sha1-5jNFY4bUqlWGP2dqerDaqP3ssP0=" + "integrity": "sha512-8dG4H5ujfvFiqDmVu9fQ5bOHUC15JMjMY/Zumv26oOvvVJjM67KF8koCWIabKQ1GJIa9r2mMZscBq/TbdOcmNA==" }, "node-forge": { "version": "0.8.5", @@ -1691,7 +1691,7 @@ "on-finished": { "version": "2.3.0", "resolved": "https://registry.npmjs.org/on-finished/-/on-finished-2.3.0.tgz", - "integrity": "sha1-IPEzZIGwg811M3mSoWlxqi2QaUc=", + "integrity": "sha512-ikqdkGAAyf/X/gPhXGvfgAytDZtDbr+bkNUJ0N9h5MI/dmdgCs3l6hoHrcUv41sRKew3jIwrp4qQDXiK99Utww==", "requires": { "ee-first": "1.1.1" } @@ -1704,7 +1704,7 @@ "once": { "version": "1.4.0", "resolved": "https://registry.npmjs.org/once/-/once-1.4.0.tgz", - "integrity": "sha1-WDsap3WWHUsROsF9nFC6753Xa9E=", + "integrity": "sha512-lNaJgI+2Q5URQBkccEKHTQOPaXdUxnZZElQTZY0MFUAuaEqe1E+Nyvgdz/aIyNi6Z9MzO5dv1H8n58/GELp3+w==", "requires": { "wrappy": "1" } @@ -1712,7 +1712,7 @@ "options": { "version": "0.0.6", "resolved": "https://registry.npmjs.org/options/-/options-0.0.6.tgz", - "integrity": "sha1-7CLTEoBrtT5zF3Pnza788cZDEo8=" + "integrity": "sha512-bOj3L1ypm++N+n7CEbbe473A414AB7z+amKYshRb//iuL3MpdDCLhPnw6aVTdKB9g5ZRVHIEp8eUln6L2NUStg==" }, "p-limit": { "version": "2.2.2", @@ -1725,7 +1725,7 @@ "p-try": { "version": "2.2.0", "resolved": "https://registry.npmjs.org/p-try/-/p-try-2.2.0.tgz", - "integrity": "sha1-yyhoVA4xPWHeWPr741zpAE1VQOY=" + "integrity": "sha512-R4nPAVTAU0B9D35/Gk3uJf/7XYbQcyohSKdvAxIRSNghFl4e71hVoGnBNQz9cWaXxO2I10KTC+3jMdvvoKw6dQ==" }, "parse-duration": { "version": "0.1.2", @@ -1735,7 +1735,7 @@ "parse-ms": { "version": "2.1.0", "resolved": "https://registry.npmjs.org/parse-ms/-/parse-ms-2.1.0.tgz", - "integrity": "sha1-NIVlp1PUOR+lJAKZVrFyy3dTCX0=" + "integrity": "sha512-kHt7kzLoS9VBZfUsiKjv43mr91ea+U05EyKkEtqp7vNbHxmaVuEqN7XxeEVnGrMtYOAxGrDElSi96K7EgO1zCA==" }, "parseurl": { "version": "1.3.3", @@ -1745,37 +1745,37 @@ "path-is-absolute": { "version": "1.0.1", "resolved": "https://registry.npmjs.org/path-is-absolute/-/path-is-absolute-1.0.1.tgz", - "integrity": "sha1-F0uSaHNVNP+8es5r9TpanhtcX18=" + "integrity": "sha512-AVbw3UJ2e9bq64vSaS9Am0fje1Pa8pbGqTTsmXfaIiMpnr5DlDhfJOuLj9Sf95ZPVDAUerDfEk88MPmPe7UCQg==" }, "path-parse": { "version": "1.0.6", "resolved": "https://registry.npmjs.org/path-parse/-/path-parse-1.0.6.tgz", - "integrity": "sha1-1i27VnlAXXLEc37FhgDp3c8G0kw=" + "integrity": "sha512-GSmOT2EbHrINBf9SR7CDELwlJ8AENk3Qn7OikK4nFYAu3Ote2+JYNVvkpAEQm3/TLNEJFD/xZJjzyxg3KBWOzw==" }, "path-to-regexp": { "version": "0.1.7", "resolved": "https://registry.npmjs.org/path-to-regexp/-/path-to-regexp-0.1.7.tgz", - "integrity": "sha1-32BBeABfUi8V60SQ5yR6G/qmf4w=" + "integrity": "sha512-5DFkuoqlv1uYQKxy8omFBeJPQcdoE07Kv2sferDCrAq1ohOU+MSDswDIbnx3YAM60qIOnYa53wBhXW0EbMonrQ==" }, "performance-now": { "version": "2.1.0", "resolved": "https://registry.npmjs.org/performance-now/-/performance-now-2.1.0.tgz", - "integrity": "sha1-Ywn04OX6kT7BxpMHrjZLSzd8nns=" + "integrity": "sha512-7EAHlyLHI56VEIdK57uwHdHKIaAGbnXPiw0yWbarQZOKaKpvUIgW0jWRVLiatnM+XXlSwsanIBH/hzGMJulMow==" }, "pify": { "version": "4.0.1", "resolved": "https://registry.npmjs.org/pify/-/pify-4.0.1.tgz", - "integrity": "sha1-SyzSXFDVmHNcUCkiJP2MbfQeMjE=" + "integrity": "sha512-uB80kBFb/tfd68bVleG9T5GGsGPjJrLAUpR5PZIrhBnIaRTQRjqdJSsIKkOP6OAIFbj7GOrcudc5pNjZ+geV2g==" }, "policyfile": { "version": "0.0.4", "resolved": "https://registry.npmjs.org/policyfile/-/policyfile-0.0.4.tgz", - "integrity": "sha1-1rgurZiueeviKOLa9ZAzEeyYLk0=" + "integrity": "sha512-UfDtlscNialXfmVEwEPm0t/5qtM0xPK025eYWd/ilv89hxLIhVQmt3QIzMHincLO2MBtZyww0386pt13J4aIhQ==" }, "pretty-ms": { "version": "4.0.0", "resolved": "https://registry.npmjs.org/pretty-ms/-/pretty-ms-4.0.0.tgz", - "integrity": "sha1-Mbr0G5T9AiJwmKqgO9YmCOsNbpI=", + "integrity": "sha512-qG66ahoLCwpLXD09ZPHSCbUWYTqdosB7SMP4OffgTgL2PBKXMuUsrk5Bwg8q4qPkjTXsKBMr+YK3Ltd/6F9s/Q==", "requires": { "parse-ms": "^2.0.0" } @@ -1842,7 +1842,7 @@ "q": { "version": "0.9.2", "resolved": "https://registry.npmjs.org/q/-/q-0.9.2.tgz", - "integrity": "sha1-I8BsRsgTKGFqrhaNPuI6Vr1D2vY=" + "integrity": "sha512-ZOxMuWPMJnsUdYhuQ9glpZwKhB4cm8ubYFy1nNCY8TkSAuZun5fd8jCDTlf2ykWnK8x9HGn1stNtLeG179DebQ==" }, "qs": { "version": "6.7.0", @@ -1852,7 +1852,7 @@ "random-bytes": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/random-bytes/-/random-bytes-1.0.0.tgz", - "integrity": "sha1-T2ih3Arli9P7lYSMMDJNt11kNgs=" + "integrity": "sha512-iv7LhNVO047HzYR3InF6pUcUsPQiHTM1Qal51DcGSuZFBil1aBBWG5eHPNek7bvILMaYJ/8RU1e8w1AMdHmLQQ==" }, "range-parser": { "version": "1.2.1", @@ -1862,7 +1862,7 @@ "raven": { "version": "1.1.3", "resolved": "https://registry.npmjs.org/raven/-/raven-1.1.3.tgz", - "integrity": "sha1-QnPBrm005CMPUbLAEEGjK5Iygio=", + "integrity": "sha512-RYov4wAaflZasWiCrZuizd3jNXxCOkW1WrXgWsGVb8kRpdHNZ+vPY27R6RhVtqzWp+DG9a5l6iP0QUPK4EgzaQ==", "requires": { "cookie": "0.3.1", "json-stringify-safe": "5.0.1", @@ -1899,7 +1899,7 @@ "redis": { "version": "0.12.1", "resolved": "https://registry.npmjs.org/redis/-/redis-0.12.1.tgz", - "integrity": "sha1-ZN92rQ/IrOuuvSoGReikj6xJGF4=" + "integrity": "sha512-DtqxdmgmVAO7aEyxaXBiUTvhQPOYznTIvmPzs9AwWZqZywM50JlFxQjFhicI+LVbaun7uwfO3izuvc1L8NlPKQ==" }, "redis-commands": { "version": "1.5.0", @@ -1909,12 +1909,12 @@ "redis-errors": { "version": "1.2.0", "resolved": "https://registry.npmjs.org/redis-errors/-/redis-errors-1.2.0.tgz", - "integrity": "sha1-62LSrbFeTq9GEMBK/hUpOEJQq60=" + "integrity": "sha512-1qny3OExCf0UvUV/5wpYKf2YwPcOqXzkwKKSmKHiE6ZMQs5heeE/c8eXK+PNllPvmjgAbfnsbpkGZWy8cBpn9w==" }, "redis-parser": { "version": "3.0.0", "resolved": "https://registry.npmjs.org/redis-parser/-/redis-parser-3.0.0.tgz", - "integrity": "sha1-tm2CjNyv5rS4pCin3vTGvKwxyLQ=", + "integrity": "sha512-DJnGAeenTdpMEH6uAJRK/uiyEIH9WVsUmoLwzudwGJUwZPp80PDBWPHXSAGNPwNvIXAbe7MSUB1zQFugFml66A==", "requires": { "redis-errors": "^1.0.0" } @@ -1922,7 +1922,7 @@ "redis-sentinel": { "version": "0.1.1", "resolved": "https://registry.npmjs.org/redis-sentinel/-/redis-sentinel-0.1.1.tgz", - "integrity": "sha1-Vj3TQduZMgMfSX+v3Td+hkj/s+U=", + "integrity": "sha512-cKtLSUzDsKmsB50J1eIV/SH11DSMiHgsm/gDPRCU5lXz5OyTSuLKWg9oc8d5n74kZwtAyRkfJP0x8vYXvlPjFQ==", "requires": { "q": "0.9.2", "redis": "0.11.x" @@ -1931,7 +1931,7 @@ "redis": { "version": "0.11.0", "resolved": "https://registry.npmjs.org/redis/-/redis-0.11.0.tgz", - "integrity": "sha1-/cAdSrTL5LO7LLKByP5WnDhX9XE=" + "integrity": "sha512-wkgzIZ9HuxJ6Sul1IW/6FG13Ecv6q8kmdHb5xo09Hu6bgWzz5qsnM06SVMpDxFNbyApaRjy8CwnmVaRMMhAMWg==" } } }, @@ -1958,7 +1958,7 @@ "coffee-script": { "version": "1.8.0", "resolved": "https://registry.npmjs.org/coffee-script/-/coffee-script-1.8.0.tgz", - "integrity": "sha1-nJ8dK0pSoADe0Vtll5FwNkgmPB0=", + "integrity": "sha512-EvLTMcu9vR6G1yfnz75yrISvhq1eBPC+pZbQhHzTiC5vXgpYIrArxQc5tB+SYfBi3souVdSZ4AZzYxI72oLXUw==", "requires": { "mkdirp": "~0.3.5" } @@ -1966,7 +1966,7 @@ "mkdirp": { "version": "0.3.5", "resolved": "https://registry.npmjs.org/mkdirp/-/mkdirp-0.3.5.tgz", - "integrity": "sha1-3j5fiWHIjHh+4TaN+EmsRBPsqNc=" + "integrity": "sha512-8OCq0De/h9ZxseqzCH8Kw/Filf5pF/vMI6+BH7Lu0jXz2pqYCjTAQRolSxRIi+Ax+oCCjlxoJMP0YQ4XlrQNHg==" } } }, @@ -2064,7 +2064,7 @@ "require-like": { "version": "0.1.2", "resolved": "https://registry.npmjs.org/require-like/-/require-like-0.1.2.tgz", - "integrity": "sha1-rW8wwTvs15cBDEaK+ndcDAprR/o=", + "integrity": "sha512-oyrU88skkMtDdauHDuKVrgR+zuItqr6/c//FXzvmxRGMexSDc6hNvJInGW3LL46n+8b50RykrvwSUIIQH2LQ5A==", "dev": true }, "resolve": { @@ -2078,7 +2078,7 @@ "retry-axios": { "version": "0.3.2", "resolved": "https://registry.npmjs.org/retry-axios/-/retry-axios-0.3.2.tgz", - "integrity": "sha1-V1fID1hbTMTEmGqi/9R6YMbTXhM=" + "integrity": "sha512-jp4YlI0qyDFfXiXGhkCOliBN1G7fRH03Nqy8YdShzGqbY5/9S2x/IR6C88ls2DFkbWuL3ASkP7QD3pVrNpPgwQ==" }, "retry-request": { "version": "4.1.1", @@ -2107,7 +2107,7 @@ "rimraf": { "version": "2.4.5", "resolved": "https://registry.npmjs.org/rimraf/-/rimraf-2.4.5.tgz", - "integrity": "sha1-7nEM5dk6j9uFb7Xqj/Di11k0sto=", + "integrity": "sha512-J5xnxTyqaiw06JjMftq7L9ouA448dw/E7dKghkP9WpKNuwmARNNg+Gk8/u5ryb9N/Yo2+z3MCwuqFK/+qPOPfQ==", "optional": true, "requires": { "glob": "^6.0.1" @@ -2116,12 +2116,12 @@ "safe-buffer": { "version": "5.1.1", "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.1.1.tgz", - "integrity": "sha1-iTMSr2myEj3vcfV4iQAWce6yyFM=" + "integrity": "sha512-kKvNJn6Mm93gAczWVJg7wH+wGYWNrDHdWvpUmHyEsgCtIwwo3bqPtV4tR5tuPaUhTOo/kvhVwd8XwwOllGYkbg==" }, "safe-json-stringify": { "version": "1.2.0", "resolved": "https://registry.npmjs.org/safe-json-stringify/-/safe-json-stringify-1.2.0.tgz", - "integrity": "sha1-NW5EvJjx+TzkXfFLzXwBzahuCv0=", + "integrity": "sha512-gH8eh2nZudPQO6TytOvbxnuhYBOvDBBLW52tz5q6X58lJcd/tkmqFR+5Z9adS8aJtURSXWThWy/xJtJwixErvg==", "optional": true }, "safer-buffer": { @@ -2138,7 +2138,7 @@ "sandboxed-module": { "version": "0.3.0", "resolved": "https://registry.npmjs.org/sandboxed-module/-/sandboxed-module-0.3.0.tgz", - "integrity": "sha1-8fvvvYCaT2kHO9B8rm/H2y6vX2o=", + "integrity": "sha512-/2IfB1wtca3eNVPXbQrb6UkhE/1pV4Wz+5CdG6DPYqeaDsYDzxglBT7/cVaqyrlRyQKdmw+uTZUTRos9FFD2PQ==", "dev": true, "requires": { "require-like": "0.1.2", @@ -2148,7 +2148,7 @@ "stack-trace": { "version": "0.0.6", "resolved": "https://registry.npmjs.org/stack-trace/-/stack-trace-0.0.6.tgz", - "integrity": "sha1-HnGb1qJin/CcGJ4Xqe+QKpT8XbA=", + "integrity": "sha512-5/6uZt7RYjjAl8z2j1mXWAewz+I4Hk2/L/3n6NRLIQ31+uQ7nMd9O6G69QCdrrufHv0QGRRHl/jwUEGTqhelTA==", "dev": true } } @@ -2204,7 +2204,7 @@ "settings-sharelatex": { "version": "1.1.0", "resolved": "https://registry.npmjs.org/settings-sharelatex/-/settings-sharelatex-1.1.0.tgz", - "integrity": "sha1-Tv4vUpPbjxwVlnEEx5BfqHD/mS0=", + "integrity": "sha512-f7D+0lnlohoteSn6IKTH72NE+JnAdMWTKwQglAuimZWTID2FRRItZSGeYMTRpvEnaQApkoVwRp//WRMsiddnqw==", "requires": { "coffee-script": "1.6.0" }, @@ -2212,14 +2212,14 @@ "coffee-script": { "version": "1.6.0", "resolved": "https://registry.npmjs.org/coffee-script/-/coffee-script-1.6.0.tgz", - "integrity": "sha1-gIs5bhEPU9AhoZpO8fZb4OjjX6M=" + "integrity": "sha512-Tx8itEfCsQp8RbLDFt7qwjqXycAx2g6SI7//4PPUR2j6meLmNifYm6zKrNDcU1+Q/GWRhjhEZk7DaLG1TfIzGA==" } } }, "shimmer": { "version": "1.2.1", "resolved": "https://registry.npmjs.org/shimmer/-/shimmer-1.2.1.tgz", - "integrity": "sha1-YQhZ994ye1h+/r9QH7QxF/mv8zc=" + "integrity": "sha512-sQTKC1Re/rM6XyFM6fIAGHRPVGvyXfgzIDvzoq608vM+jeyVD0Tu1E6Np0Kc2zAIFWIj963V2800iF/9LPieQw==" }, "sinon": { "version": "2.4.1", @@ -2240,7 +2240,7 @@ "isarray": { "version": "0.0.1", "resolved": "https://registry.npmjs.org/isarray/-/isarray-0.0.1.tgz", - "integrity": "sha1-ihis/Kmo9Bd+Cav8YDiTmwXR7t8=", + "integrity": "sha512-D2S+3GLxWH+uhrNEcoh/fnmYeP8E8/zHl644d/jdA0g2uyXvy3sb0qxotE+ne0LtccHknQzWwZEzhak7oJ0COQ==", "dev": true }, "path-to-regexp": { @@ -2263,7 +2263,7 @@ "socket.io": { "version": "0.9.19", "resolved": "https://registry.npmjs.org/socket.io/-/socket.io-0.9.19.tgz", - "integrity": "sha1-SQu1/Q3FTPAC7gTmf638Q7hIo48=", + "integrity": "sha512-UPdVIGPBPmCibzIP2rAjXuiPTI2gPs6kiu4P7njH6WAK7wiOlozNG62ohohCNOycx+Dztd4vRNXxq8alIOEtfA==", "requires": { "base64id": "0.1.0", "policyfile": "0.0.4", @@ -2274,13 +2274,13 @@ "redis": { "version": "0.7.3", "resolved": "https://registry.npmjs.org/redis/-/redis-0.7.3.tgz", - "integrity": "sha1-7le3pE0l7BWU5ENl2BZfp9HUgRo=", + "integrity": "sha512-0Pgb0jOLfn6eREtEIRn/ifyZJjl2H+wUY4F/Pe7T4UhmoSrZ/1HU5ZqiBpDk8I8Wbyv2N5DpXKzbEtMj3drprg==", "optional": true }, "socket.io-client": { "version": "0.9.16", "resolved": "https://registry.npmjs.org/socket.io-client/-/socket.io-client-0.9.16.tgz", - "integrity": "sha1-TadRXF53MEHRtCOXBBW8xDDzX8Y=", + "integrity": "sha512-wSM7PKJkzpGqUAo6d6SAn+ph4xeQJ6nzyDULRJAX1G7e6Xm0wNuMh2RpNvwXrHMzoV9Or5hti7LINiQAm1H2yA==", "requires": { "active-x-obfuscator": "0.0.1", "uglify-js": "1.2.5", @@ -2293,7 +2293,7 @@ "socket.io-client": { "version": "0.9.17", "resolved": "https://registry.npmjs.org/socket.io-client/-/socket.io-client-0.9.17.tgz", - "integrity": "sha1-MdEsRLc/u6NJY57M+DPj+a4T/m8=", + "integrity": "sha512-gKV451FUZPLeJmA8vPfvqRZctAzWNOlaB0C06MeDFmrGquDNMllnpXp+1+4QS2NaZvcycoJHVt72R5uNaLCIBg==", "requires": { "active-x-obfuscator": "0.0.1", "uglify-js": "1.2.5", @@ -2304,12 +2304,12 @@ "source-map": { "version": "0.6.1", "resolved": "https://registry.npmjs.org/source-map/-/source-map-0.6.1.tgz", - "integrity": "sha1-dHIq8y6WFOnCh6jQu95IteLxomM=" + "integrity": "sha512-UjgapumWlbMhkBgzT7Ykc5YXUT46F0iKu8SGXq0bcwP5dz/h0Plj6enJqjz1Zbq2l5WaqYnrVbwWOWMyF3F47g==" }, "split": { "version": "1.0.1", "resolved": "https://registry.npmjs.org/split/-/split-1.0.1.tgz", - "integrity": "sha1-YFvZvjA6pZ+zX5Ip++oN3snqB9k=", + "integrity": "sha512-mTyOoPbrivtXnwnIxZRFYRrPNtEFKlpB2fvjSnCQUiAA6qAZzqwna5envK4uk6OIeP17CsdF3rSBGYVBsU0Tkg==", "requires": { "through": "2" } @@ -2333,7 +2333,7 @@ "stack-trace": { "version": "0.0.9", "resolved": "https://registry.npmjs.org/stack-trace/-/stack-trace-0.0.9.tgz", - "integrity": "sha1-qPbq7KkGdMMz58Q5U/J1tFFRBpU=" + "integrity": "sha512-vjUc6sfgtgY0dxCdnc40mK6Oftjo9+2K8H/NG81TMhgL392FtiPA9tn9RLyTxXmTLPJPjF3VyzFp6bsWFLisMQ==" }, "standard-as-callback": { "version": "2.0.1", @@ -2343,12 +2343,12 @@ "statsd-parser": { "version": "0.0.4", "resolved": "https://registry.npmjs.org/statsd-parser/-/statsd-parser-0.0.4.tgz", - "integrity": "sha1-y9JDlTzELv/VSLXSI4jtaJ7GOb0=" + "integrity": "sha512-7XO+ur89EalMXXFQaydsczB8sclr5nDsNIoUu0IzJx1pIbHUhO3LtpSzBwetIuU9DyTLMiVaJBMtWS/Nb2KR4g==" }, "statuses": { "version": "1.5.0", "resolved": "https://registry.npmjs.org/statuses/-/statuses-1.5.0.tgz", - "integrity": "sha1-Fhx9rBd2Wf2YEfQ3cfqZOBR4Yow=" + "integrity": "sha512-OpZ3zP+jT1PI7I8nemJX4AKmAX070ZkYPVWV/AaKTJl+tXCTGyVdC1a4SL8RUQYEwk/f34ZX8UTykN68FwrqAA==" }, "stream-shift": { "version": "1.0.1", @@ -2358,7 +2358,7 @@ "string_decoder": { "version": "1.1.1", "resolved": "https://registry.npmjs.org/string_decoder/-/string_decoder-1.1.1.tgz", - "integrity": "sha1-nPFhG6YmhdcDCunkujQUnDrwP8g=", + "integrity": "sha512-n/ShnvDi6FHbbVfviro+WojiFzv+s8MPMHBczVePfUpDJLwoLT0ht1l4YwBCbi8pJAveEEdnkHyPyTP/mzRfwg==", "requires": { "safe-buffer": "~5.1.0" } @@ -2366,7 +2366,7 @@ "tdigest": { "version": "0.1.1", "resolved": "https://registry.npmjs.org/tdigest/-/tdigest-0.1.1.tgz", - "integrity": "sha1-Ljyyw56kSeVdHmzZEReszKRYgCE=", + "integrity": "sha512-CXcDY/NIgIbKZPx5H4JJNpq6JwJhU5Z4+yWj4ZghDc7/9nVajiRlPPyMXRePPPlBfcayUqtoCXjo7/Hm82ecUA==", "requires": { "bintrees": "1.0.1" } @@ -2374,7 +2374,7 @@ "teeny-request": { "version": "3.11.3", "resolved": "https://registry.npmjs.org/teeny-request/-/teeny-request-3.11.3.tgz", - "integrity": "sha1-M1xin3ZF5dZZk2LfLzIwxMvCOlU=", + "integrity": "sha512-CKncqSF7sH6p4rzCgkb/z/Pcos5efl0DmolzvlqRQUNcpRIruOhY9+T1FsIlyEbfWd7MsFpodROOwHYh2BaXzw==", "requires": { "https-proxy-agent": "^2.2.1", "node-fetch": "^2.2.0", @@ -2391,13 +2391,13 @@ "text-encoding": { "version": "0.6.4", "resolved": "https://registry.npmjs.org/text-encoding/-/text-encoding-0.6.4.tgz", - "integrity": "sha1-45mpgiV6J22uQou5KEXLcb3CbRk=", + "integrity": "sha512-hJnc6Qg3dWoOMkqP53F0dzRIgtmsAge09kxUIqGrEUS4qr5rWLckGYaQAVr+opBrIMRErGgy6f5aPnyPpyGRfg==", "dev": true }, "through": { "version": "2.3.8", "resolved": "https://registry.npmjs.org/through/-/through-2.3.8.tgz", - "integrity": "sha1-DdTJ/6q8NXlgsbckEV1+Doai4fU=" + "integrity": "sha512-w89qg7PI8wAdvX60bMDP+bFoD5Dvhm9oLheFp5O4a2QF0cSBGsBX4qZmadPMvVqlLJBBci+WqGGOAPvcDeNSVg==" }, "through2": { "version": "3.0.1", @@ -2410,13 +2410,13 @@ "timekeeper": { "version": "0.0.4", "resolved": "https://registry.npmjs.org/timekeeper/-/timekeeper-0.0.4.tgz", - "integrity": "sha1-kNt58X2Ni1NiFUOJSSuXJ2LP0nY=", + "integrity": "sha512-QSNovcsPIbrI9zzXTesL/iiDrS+4IT+0xCxFzDSI2/yHkHVL1QEB5FhzrKMzCEwsbSOISEsH1yPUZ6/Fve9DZQ==", "dev": true }, "tinycolor": { "version": "0.0.1", "resolved": "https://registry.npmjs.org/tinycolor/-/tinycolor-0.0.1.tgz", - "integrity": "sha1-MgtaUtg6u1l42Bo+iH1K77FaYWQ=" + "integrity": "sha512-+CorETse1kl98xg0WAzii8DTT4ABF4R3nquhrkIbVGcw1T8JYs5Gfx9xEfGINPUZGDj9C4BmOtuKeaTtuuRolg==" }, "toidentifier": { "version": "1.0.0", @@ -2435,14 +2435,14 @@ "punycode": { "version": "1.4.1", "resolved": "https://registry.npmjs.org/punycode/-/punycode-1.4.1.tgz", - "integrity": "sha1-wNWmOycYgArY4esPpSachN1BhF4=" + "integrity": "sha512-jmYNElW7yvO7TV33CjSmvSiE2yco3bV2czu/OzDKdMNVZQWfxCblURLhf+47syQRBntjfLdd/H0egrzIG+oaFQ==" } } }, "tunnel-agent": { "version": "0.6.0", "resolved": "https://registry.npmjs.org/tunnel-agent/-/tunnel-agent-0.6.0.tgz", - "integrity": "sha1-J6XeoGs2sEoKmWZ3SykIaPD8QP0=", + "integrity": "sha512-McnNiV1l8RYeY8tBgEpuodCC1mLUdbSN+CYBL7kJsJNInOP8UjDDEwdk6Mw60vdLLrr5NHKZhMAOSrR2NZuQ+w==", "requires": { "safe-buffer": "^5.0.1" } @@ -2450,12 +2450,12 @@ "tweetnacl": { "version": "0.14.5", "resolved": "https://registry.npmjs.org/tweetnacl/-/tweetnacl-0.14.5.tgz", - "integrity": "sha1-WuaBd/GS1EViadEIr6k/+HQ/T2Q=" + "integrity": "sha512-KXXFFdAbFXY4geFIwoyNK+f5Z1b7swfXABfL7HXCmoIWMKU3dmS26672A4EeQtDzLKy7SXmfBu51JolvEKwtGA==" }, "type-detect": { "version": "0.1.1", "resolved": "https://registry.npmjs.org/type-detect/-/type-detect-0.1.1.tgz", - "integrity": "sha1-C6XsKohWQORw6k6FBZcZANrFiCI=", + "integrity": "sha512-5rqszGVwYgBoDkIm2oUtvkfZMQ0vk29iDMU0W2qCa3rG0vPDNczCMT4hV/bLBgLg8k8ri6+u3Zbt+S/14eMzlA==", "dev": true }, "type-is": { @@ -2485,7 +2485,7 @@ "uglify-js": { "version": "1.2.5", "resolved": "https://registry.npmjs.org/uglify-js/-/uglify-js-1.2.5.tgz", - "integrity": "sha1-tULCx29477NLIAsgF3Y0Mw/3ArY=" + "integrity": "sha512-Ps1oQryKOcRDYuAN1tGpPWd/DIRMcdLz4p7JMxLjJiFvp+aaG01IEu0ZSoVvYUSxIkvW7k2X50BCW2InguEGlg==" }, "uid-safe": { "version": "2.1.5", @@ -2498,12 +2498,12 @@ "underscore": { "version": "1.7.0", "resolved": "https://registry.npmjs.org/underscore/-/underscore-1.7.0.tgz", - "integrity": "sha1-a7rwh3UA02vjTsqlhODbn+8DUgk=" + "integrity": "sha512-cp0oQQyZhUM1kpJDLdGO1jPZHgS/MpzoWYfe9+CM2h/QGDZlqwT2T3YGukuBdaNJ/CAPoeyAZRRHz8JFo176vA==" }, "unpipe": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/unpipe/-/unpipe-1.0.0.tgz", - "integrity": "sha1-sr9O6FFKrmFltIF4KdIbLvSZBOw=" + "integrity": "sha512-pjy2bYhSsufwWlKwPc+l3cN7+wuJlK6uz0YdJEOlQDbl6jo/YlPi4mb8agUkVC8BF7V8NuzeyPNqRksA3hztKQ==" }, "uri-js": { "version": "4.2.2", @@ -2516,27 +2516,27 @@ "util-deprecate": { "version": "1.0.2", "resolved": "https://registry.npmjs.org/util-deprecate/-/util-deprecate-1.0.2.tgz", - "integrity": "sha1-RQ1Nyfpw3nMnYvvS1KKJgUGaDM8=" + "integrity": "sha512-EPD5q1uXyFxJpCrLnCc1nHnq3gOa6DZBocAIiI2TaSCA7VCJ1UJDMagCzIkXNsUYfD1daK//LTEQ8xiIbrHtcw==" }, "utils-merge": { "version": "1.0.1", "resolved": "https://registry.npmjs.org/utils-merge/-/utils-merge-1.0.1.tgz", - "integrity": "sha1-n5VxD1CiZ5R7LMwSR0HBAoQn5xM=" + "integrity": "sha512-pMZTvIkT1d+TFGvDOqodOclx0QWkkgi6Tdoa8gC8ffGAAqz9pzPTZWAybbsHHoED/ztMtkv/VoYTYyShUn81hA==" }, "uuid": { "version": "3.0.0", "resolved": "https://registry.npmjs.org/uuid/-/uuid-3.0.0.tgz", - "integrity": "sha1-Zyj8BFnEUNeWqZwxg3VpvfZy1yg=" + "integrity": "sha512-rqE1LoOVLv3QrZMjb4NkF5UWlkurCfPyItVnFPNKDDGkHw4dQUdE4zMcLqx28+0Kcf3+bnUk4PisaiRJT4aiaQ==" }, "vary": { "version": "1.1.2", "resolved": "https://registry.npmjs.org/vary/-/vary-1.1.2.tgz", - "integrity": "sha1-IpnwLG3tMNSllhsLn3RSShj2NPw=" + "integrity": "sha512-BNGbWLfd0eUPabhkXUVm0j8uuvREyTh5ovRa/dyow/BqAbZJyC+5fU+IzQOzmAKzYqYRAISoRhdQr3eIZ/PXqg==" }, "verror": { "version": "1.10.0", "resolved": "https://registry.npmjs.org/verror/-/verror-1.10.0.tgz", - "integrity": "sha1-OhBcoXBTr1XW4nDB+CiGguGNpAA=", + "integrity": "sha512-ZZKSmDAEFOijERBLkmYfJ+vmk3w+7hOLYDNkRCuRuMJGEmqYNCNLyBBFwWKVMhfwaEF3WOd0Zlw86U/WC/+nYw==", "requires": { "assert-plus": "^1.0.0", "core-util-is": "1.0.2", @@ -2546,12 +2546,12 @@ "wrappy": { "version": "1.0.2", "resolved": "https://registry.npmjs.org/wrappy/-/wrappy-1.0.2.tgz", - "integrity": "sha1-tSQ9jz7BqjXxNkYFvA0QNuMKtp8=" + "integrity": "sha512-l4Sp/DRseor9wL6EvV2+TuQn63dMkPjZ/sp9XkghTEbV9KlPS1xUsZ3u7/IQO4wxtcFB4bgpQPRcR3QCvezPcQ==" }, "ws": { "version": "0.4.32", "resolved": "https://registry.npmjs.org/ws/-/ws-0.4.32.tgz", - "integrity": "sha1-eHphVEFPPJntg8V3IVOyD+sM7DI=", + "integrity": "sha512-htqsS0U9Z9lb3ITjidQkRvkLdVhQePrMeu475yEfOWkAYvJ6dSjQp1tOH6ugaddzX5b7sQjMPNtY71eTzrV/kA==", "requires": { "commander": "~2.1.0", "nan": "~1.0.0", @@ -2562,19 +2562,19 @@ "commander": { "version": "2.1.0", "resolved": "https://registry.npmjs.org/commander/-/commander-2.1.0.tgz", - "integrity": "sha1-0SG7roYNmZKj1Re6lvVliOR8Z4E=" + "integrity": "sha512-J2wnb6TKniXNOtoHS8TSrG9IOQluPrsmyAJ8oCUJOBmv+uLBCyPYAZkD2jFvw2DCzIXNnISIM01NIvr35TkBMQ==" }, "nan": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/nan/-/nan-1.0.0.tgz", - "integrity": "sha1-riT4hQgY1mL8q1rPfzuVv6oszzg=" + "integrity": "sha512-Wm2/nFOm2y9HtJfgOLnctGbfvF23FcQZeyUZqDD8JQG3zO5kXh3MkQKiUaA68mJiVWrOzLFkAV1u6bC8P52DJA==" } } }, "xmlhttprequest": { "version": "1.4.2", "resolved": "https://registry.npmjs.org/xmlhttprequest/-/xmlhttprequest-1.4.2.tgz", - "integrity": "sha1-AUU6HZvtHo8XL2SVu/TIxCYyFQA=" + "integrity": "sha512-WTsthd44hTdCRrHkdtTgbgTKIJyNDV+xiShdooFZBUstY7xk+EXMx/u5gjuUXaCiCWvtBVCHwauzml2joevB4w==" }, "yallist": { "version": "3.1.1", @@ -2589,7 +2589,7 @@ "zeparser": { "version": "0.0.5", "resolved": "https://registry.npmjs.org/zeparser/-/zeparser-0.0.5.tgz", - "integrity": "sha1-A3JlYbwmjy5URPVMZlt/1KjAKeI=" + "integrity": "sha512-Qj4lJIRPy7hIW1zCBqwA3AW8F9uHswVoXPnotuY6uyNgbg5qGb6SJfWZi+YzD3DktbUnUoGiGZFhopbn9l1GYw==" } } } From 3ffc218b2c18357007f21b0ce6438f42acfb23ef Mon Sep 17 00:00:00 2001 From: Miguel Serrano Date: Mon, 22 Jun 2020 13:13:44 +0200 Subject: [PATCH 358/491] Rolled back redis-sharelatex to 1.0.12 --- services/real-time/package-lock.json | 8 ++++---- services/real-time/package.json | 2 +- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/services/real-time/package-lock.json b/services/real-time/package-lock.json index 72c96d7df6..b6b2a7a811 100644 --- a/services/real-time/package-lock.json +++ b/services/real-time/package-lock.json @@ -1936,13 +1936,13 @@ } }, "redis-sharelatex": { - "version": "1.0.13", - "resolved": "https://registry.npmjs.org/redis-sharelatex/-/redis-sharelatex-1.0.13.tgz", - "integrity": "sha512-sAQNofqfcMlIxzxNJF1qUspJKDM1VuuIOrGZQX9nb5JtcJ5cusa5sc+Oyb51eymPV5mZGWT3u07tKtv4jdXVIg==", + "version": "1.0.12", + "resolved": "https://registry.npmjs.org/redis-sharelatex/-/redis-sharelatex-1.0.12.tgz", + "integrity": "sha512-Z+LDGaRNgZ+NiDaCC/R0N3Uy6SCtbKXqiXlvCwAbIQRSZUc69OVx/cQ3i5qDF7zeERhh+pnTd+zGs8nVfa5p+Q==", "requires": { "async": "^2.5.0", "coffee-script": "1.8.0", - "ioredis": "~4.17.3", + "ioredis": "~4.16.1", "redis-sentinel": "0.1.1", "underscore": "1.7.0" }, diff --git a/services/real-time/package.json b/services/real-time/package.json index 157895473e..b6b2cfb8b1 100644 --- a/services/real-time/package.json +++ b/services/real-time/package.json @@ -31,7 +31,7 @@ "express-session": "^1.17.1", "logger-sharelatex": "^1.7.0", "metrics-sharelatex": "^2.6.2", - "redis-sharelatex": "^1.0.13", + "redis-sharelatex": "^1.0.12", "request": "^2.88.2", "settings-sharelatex": "^1.1.0", "socket.io": "0.9.19", From 5282f8f5314c8ab4309c844ea6d34e4ec95dff09 Mon Sep 17 00:00:00 2001 From: Jakob Ackermann Date: Mon, 24 Feb 2020 13:32:20 +0000 Subject: [PATCH 359/491] [misc] synchronous client store using an Object at .ol_context --- .../app/coffee/AuthorizationManager.coffee | 27 +++--- .../app/coffee/HttpController.coffee | 8 +- services/real-time/app/coffee/Router.coffee | 17 ++-- services/real-time/app/coffee/Utils.coffee | 14 --- .../app/coffee/WebsocketController.coffee | 61 ++++++------- .../app/coffee/WebsocketLoadBalancer.coffee | 4 +- .../coffee/AuthorizationManagerTests.coffee | 38 ++++---- .../coffee/WebsocketControllerTests.coffee | 86 ++++++++----------- .../coffee/WebsocketLoadBalancerTests.coffee | 28 +++--- .../unit/coffee/helpers/MockClient.coffee | 7 +- 10 files changed, 113 insertions(+), 177 deletions(-) delete mode 100644 services/real-time/app/coffee/Utils.coffee diff --git a/services/real-time/app/coffee/AuthorizationManager.coffee b/services/real-time/app/coffee/AuthorizationManager.coffee index b4cf854238..50d76537ce 100644 --- a/services/real-time/app/coffee/AuthorizationManager.coffee +++ b/services/real-time/app/coffee/AuthorizationManager.coffee @@ -6,13 +6,10 @@ module.exports = AuthorizationManager = AuthorizationManager._assertClientHasPrivilegeLevel client, ["readAndWrite", "owner"], callback _assertClientHasPrivilegeLevel: (client, allowedLevels, callback = (error) ->) -> - client.get "privilege_level", (error, privilegeLevel) -> - return callback(error) if error? - allowed = (privilegeLevel in allowedLevels) - if allowed - callback null - else - callback new Error("not authorized") + if client.ol_context["privilege_level"] in allowedLevels + callback null + else + callback new Error("not authorized") assertClientCanViewProjectAndDoc: (client, doc_id, callback = (error) ->) -> AuthorizationManager.assertClientCanViewProject client, (error) -> @@ -25,15 +22,15 @@ module.exports = AuthorizationManager = AuthorizationManager._assertClientCanAccessDoc client, doc_id, callback _assertClientCanAccessDoc: (client, doc_id, callback = (error) ->) -> - client.get "doc:#{doc_id}", (error, status) -> - return callback(error) if error? - if status? and status is "allowed" - callback null - else - callback new Error("not authorized") + if client.ol_context["doc:#{doc_id}"] is "allowed" + callback null + else + callback new Error("not authorized") addAccessToDoc: (client, doc_id, callback = (error) ->) -> - client.set("doc:#{doc_id}", "allowed", callback) + client.ol_context["doc:#{doc_id}"] = "allowed" + callback(null) removeAccessToDoc: (client, doc_id, callback = (error) ->) -> - client.del("doc:#{doc_id}", callback) + delete client.ol_context["doc:#{doc_id}"] + callback(null) diff --git a/services/real-time/app/coffee/HttpController.coffee b/services/real-time/app/coffee/HttpController.coffee index ae92a9e299..1fc74e8c16 100644 --- a/services/real-time/app/coffee/HttpController.coffee +++ b/services/real-time/app/coffee/HttpController.coffee @@ -1,4 +1,3 @@ -Utils = require "./Utils" async = require "async" module.exports = HttpController = @@ -8,11 +7,8 @@ module.exports = HttpController = # and for checking internal state in acceptance tests. The acceptances tests # should provide appropriate coverage. _getConnectedClientView: (ioClient, callback = (error, client) ->) -> - client_id = ioClient.id - Utils.getClientAttributes ioClient, [ - "project_id", "user_id", "first_name", "last_name", "email", "connected_time" - ], (error, {project_id, user_id, first_name, last_name, email, connected_time}) -> - return callback(error) if error? + client_id = ioClient.id + {project_id, user_id, first_name, last_name, email, connected_time} = ioClient.ol_context client = {client_id, project_id, user_id, first_name, last_name, email, connected_time} client.rooms = [] for name, joined of ioClient.manager.roomClients[client_id] diff --git a/services/real-time/app/coffee/Router.coffee b/services/real-time/app/coffee/Router.coffee index 49166b8de1..f89e1d9eaf 100644 --- a/services/real-time/app/coffee/Router.coffee +++ b/services/real-time/app/coffee/Router.coffee @@ -4,7 +4,6 @@ settings = require "settings-sharelatex" WebsocketController = require "./WebsocketController" HttpController = require "./HttpController" HttpApiController = require "./HttpApiController" -Utils = require "./Utils" bodyParser = require "body-parser" base64id = require("base64id") @@ -16,10 +15,9 @@ httpAuth = basicAuth (user, pass)-> return isValid module.exports = Router = - _handleError: (callback = ((error) ->), error, client, method, extraAttrs = {}) -> - Utils.getClientAttributes client, ["project_id", "doc_id", "user_id"], (_, attrs) -> - for key, value of extraAttrs - attrs[key] = value + _handleError: (callback = ((error) ->), error, client, method, attrs = {}) -> + for key in ["project_id", "doc_id", "user_id"] + attrs[key] = client.ol_context[key] attrs.client_id = client.id attrs.err = error if error.name == "CodedError" @@ -57,6 +55,8 @@ module.exports = Router = app.post "/client/:client_id/disconnect", httpAuth, HttpApiController.disconnectClient session.on 'connection', (error, client, session) -> + client.ol_context = {} + client?.on "error", (err) -> logger.err { clientErr: err }, "socket.io client error" if client.connected @@ -112,9 +112,14 @@ module.exports = Router = client.on "disconnect", () -> metrics.inc('socket-io.disconnect') metrics.gauge('socket-io.clients', io.sockets.clients()?.length - 1) + + cleanup = () -> + delete client.ol_context WebsocketController.leaveProject io, client, (err) -> if err? - Router._handleError null, err, client, "leaveProject" + Router._handleError cleanup, err, client, "leaveProject" + else + cleanup() # Variadic. The possible arguments: # doc_id, callback diff --git a/services/real-time/app/coffee/Utils.coffee b/services/real-time/app/coffee/Utils.coffee deleted file mode 100644 index 72dada30a3..0000000000 --- a/services/real-time/app/coffee/Utils.coffee +++ /dev/null @@ -1,14 +0,0 @@ -async = require "async" - -module.exports = Utils = - getClientAttributes: (client, keys, callback = (error, attributes) ->) -> - attributes = {} - jobs = keys.map (key) -> - (callback) -> - client.get key, (error, value) -> - return callback(error) if error? - attributes[key] = value - callback() - async.series jobs, (error) -> - return callback(error) if error? - callback null, attributes \ No newline at end of file diff --git a/services/real-time/app/coffee/WebsocketController.coffee b/services/real-time/app/coffee/WebsocketController.coffee index cdcf42a147..3ca030ca0c 100644 --- a/services/real-time/app/coffee/WebsocketController.coffee +++ b/services/real-time/app/coffee/WebsocketController.coffee @@ -7,7 +7,6 @@ DocumentUpdaterManager = require "./DocumentUpdaterManager" ConnectedUsersManager = require "./ConnectedUsersManager" WebsocketLoadBalancer = require "./WebsocketLoadBalancer" RoomManager = require "./RoomManager" -Utils = require "./Utils" module.exports = WebsocketController = # If the protocol version changes when the client reconnects, @@ -34,17 +33,17 @@ module.exports = WebsocketController = logger.warn {err, project_id, user_id, client_id: client.id}, "user is not authorized to join project" return callback(err) - client.set("privilege_level", privilegeLevel) - client.set("user_id", user_id) - client.set("project_id", project_id) - client.set("owner_id", project?.owner?._id) - client.set("first_name", user?.first_name) - client.set("last_name", user?.last_name) - client.set("email", user?.email) - client.set("connected_time", new Date()) - client.set("signup_date", user?.signUpDate) - client.set("login_count", user?.loginCount) - client.set("is_restricted_user", !!(isRestrictedUser)) + client.ol_context["privilege_level"] = privilegeLevel + client.ol_context["user_id"] = user_id + client.ol_context["project_id"] = project_id + client.ol_context["owner_id"] = project?.owner?._id + client.ol_context["first_name"] = user?.first_name + client.ol_context["last_name"] = user?.last_name + client.ol_context["email"] = user?.email + client.ol_context["connected_time"] = new Date() + client.ol_context["signup_date"] = user?.signUpDate + client.ol_context["login_count"] = user?.loginCount + client.ol_context["is_restricted_user"] = !!(isRestrictedUser) RoomManager.joinProject client, project_id, (err) -> return callback(err) if err @@ -59,8 +58,7 @@ module.exports = WebsocketController = # is determined by FLUSH_IF_EMPTY_DELAY. FLUSH_IF_EMPTY_DELAY: 500 #ms leaveProject: (io, client, callback = (error) ->) -> - Utils.getClientAttributes client, ["project_id", "user_id"], (error, {project_id, user_id}) -> - return callback(error) if error? + {project_id, user_id} = client.ol_context return callback() unless project_id # client did not join project metrics.inc "editor.leave-project" @@ -84,13 +82,12 @@ module.exports = WebsocketController = , WebsocketController.FLUSH_IF_EMPTY_DELAY joinDoc: (client, doc_id, fromVersion = -1, options, callback = (error, doclines, version, ops, ranges) ->) -> - if client.disconnected + if client.disconnected metrics.inc('editor.join-doc.disconnected', 1, {status: 'immediately'}) return callback() - metrics.inc "editor.join-doc" - Utils.getClientAttributes client, ["project_id", "user_id", "is_restricted_user"], (error, {project_id, user_id, is_restricted_user}) -> - return callback(error) if error? + metrics.inc "editor.join-doc" + {project_id, user_id, is_restricted_user} = client.ol_context return callback(new Error("no project_id found on client")) if !project_id? logger.log {user_id, project_id, doc_id, fromVersion, client_id: client.id}, "client joining doc" @@ -142,9 +139,9 @@ module.exports = WebsocketController = callback null, escapedLines, version, ops, ranges leaveDoc: (client, doc_id, callback = (error) ->) -> - # client may have disconnected, but we have to cleanup internal state. - metrics.inc "editor.leave-doc" - Utils.getClientAttributes client, ["project_id", "user_id"], (error, {project_id, user_id}) -> + # client may have disconnected, but we have to cleanup internal state. + metrics.inc "editor.leave-doc" + {project_id, user_id} = client.ol_context logger.log {user_id, project_id, doc_id, client_id: client.id}, "client leaving doc" RoomManager.leaveDoc(client, doc_id) # we could remove permission when user leaves a doc, but because @@ -153,15 +150,12 @@ module.exports = WebsocketController = ## AuthorizationManager.removeAccessToDoc client, doc_id callback() updateClientPosition: (client, cursorData, callback = (error) ->) -> - if client.disconnected + if client.disconnected # do not create a ghost entry in redis return callback() - metrics.inc "editor.update-client-position", 0.1 - Utils.getClientAttributes client, [ - "project_id", "first_name", "last_name", "email", "user_id" - ], (error, {project_id, first_name, last_name, email, user_id}) -> - return callback(error) if error? + metrics.inc "editor.update-client-position", 0.1 + {project_id, first_name, last_name, email, user_id} = client.ol_context logger.log {user_id, project_id, client_id: client.id, cursorData: cursorData}, "updating client position" AuthorizationManager.assertClientCanViewProjectAndDoc client, cursorData.doc_id, (error) -> @@ -198,14 +192,12 @@ module.exports = WebsocketController = CLIENT_REFRESH_DELAY: 1000 getConnectedUsers: (client, callback = (error, users) ->) -> - if client.disconnected + if client.disconnected # they are not interested anymore, skip the redis lookups return callback() - metrics.inc "editor.get-connected-users" - Utils.getClientAttributes client, ["project_id", "user_id", "is_restricted_user"], (error, clientAttributes) -> - return callback(error) if error? - {project_id, user_id, is_restricted_user} = clientAttributes + metrics.inc "editor.get-connected-users" + {project_id, user_id, is_restricted_user} = client.ol_context if is_restricted_user return callback(null, []) return callback(new Error("no project_id found on client")) if !project_id? @@ -221,9 +213,8 @@ module.exports = WebsocketController = , WebsocketController.CLIENT_REFRESH_DELAY applyOtUpdate: (client, doc_id, update, callback = (error) ->) -> - # client may have disconnected, but we can submit their update to doc-updater anyways. - Utils.getClientAttributes client, ["user_id", "project_id"], (error, {user_id, project_id}) -> - return callback(error) if error? + # client may have disconnected, but we can submit their update to doc-updater anyways. + {user_id, project_id} = client.ol_context return callback(new Error("no project_id found on client")) if !project_id? WebsocketController._assertClientCanApplyUpdate client, doc_id, update, (error) -> diff --git a/services/real-time/app/coffee/WebsocketLoadBalancer.coffee b/services/real-time/app/coffee/WebsocketLoadBalancer.coffee index 9508ac8714..5a59ea504f 100644 --- a/services/real-time/app/coffee/WebsocketLoadBalancer.coffee +++ b/services/real-time/app/coffee/WebsocketLoadBalancer.coffee @@ -7,7 +7,6 @@ HealthCheckManager = require "./HealthCheckManager" RoomManager = require "./RoomManager" ChannelManager = require "./ChannelManager" ConnectedUsersManager = require "./ConnectedUsersManager" -Utils = require './Utils' Async = require 'async' RESTRICTED_USER_MESSAGE_TYPE_PASS_LIST = [ @@ -94,8 +93,7 @@ module.exports = WebsocketLoadBalancer = Async.eachLimit clientList , 2 , (client, cb) -> - Utils.getClientAttributes client, ['is_restricted_user'], (err, {is_restricted_user}) -> - return cb(err) if err? + is_restricted_user = client.ol_context['is_restricted_user'] if !seen[client.id] seen[client.id] = true if !(is_restricted_user && message.message not in RESTRICTED_USER_MESSAGE_TYPE_PASS_LIST) diff --git a/services/real-time/test/unit/coffee/AuthorizationManagerTests.coffee b/services/real-time/test/unit/coffee/AuthorizationManagerTests.coffee index 46b9e8be9a..143218d8b2 100644 --- a/services/real-time/test/unit/coffee/AuthorizationManagerTests.coffee +++ b/services/real-time/test/unit/coffee/AuthorizationManagerTests.coffee @@ -9,64 +9,56 @@ modulePath = '../../../app/js/AuthorizationManager' describe 'AuthorizationManager', -> beforeEach -> @client = - params: {} - get: (param, cb) -> - cb null, @params[param] - set: (param, value, cb) -> - @params[param] = value - cb() - del: (param, cb) -> - delete @params[param] - cb() + ol_context: {} @AuthorizationManager = SandboxedModule.require modulePath, requires: {} describe "assertClientCanViewProject", -> it "should allow the readOnly privilegeLevel", (done) -> - @client.params.privilege_level = "readOnly" + @client.ol_context.privilege_level = "readOnly" @AuthorizationManager.assertClientCanViewProject @client, (error) -> expect(error).to.be.null done() it "should allow the readAndWrite privilegeLevel", (done) -> - @client.params.privilege_level = "readAndWrite" + @client.ol_context.privilege_level = "readAndWrite" @AuthorizationManager.assertClientCanViewProject @client, (error) -> expect(error).to.be.null done() it "should allow the owner privilegeLevel", (done) -> - @client.params.privilege_level = "owner" + @client.ol_context.privilege_level = "owner" @AuthorizationManager.assertClientCanViewProject @client, (error) -> expect(error).to.be.null done() it "should return an error with any other privilegeLevel", (done) -> - @client.params.privilege_level = "unknown" + @client.ol_context.privilege_level = "unknown" @AuthorizationManager.assertClientCanViewProject @client, (error) -> error.message.should.equal "not authorized" done() describe "assertClientCanEditProject", -> it "should not allow the readOnly privilegeLevel", (done) -> - @client.params.privilege_level = "readOnly" + @client.ol_context.privilege_level = "readOnly" @AuthorizationManager.assertClientCanEditProject @client, (error) -> error.message.should.equal "not authorized" done() it "should allow the readAndWrite privilegeLevel", (done) -> - @client.params.privilege_level = "readAndWrite" + @client.ol_context.privilege_level = "readAndWrite" @AuthorizationManager.assertClientCanEditProject @client, (error) -> expect(error).to.be.null done() it "should allow the owner privilegeLevel", (done) -> - @client.params.privilege_level = "owner" + @client.ol_context.privilege_level = "owner" @AuthorizationManager.assertClientCanEditProject @client, (error) -> expect(error).to.be.null done() it "should return an error with any other privilegeLevel", (done) -> - @client.params.privilege_level = "unknown" + @client.ol_context.privilege_level = "unknown" @AuthorizationManager.assertClientCanEditProject @client, (error) -> error.message.should.equal "not authorized" done() @@ -77,11 +69,11 @@ describe 'AuthorizationManager', -> beforeEach () -> @doc_id = "12345" @callback = sinon.stub() - @client.params = {} + @client.ol_context = {} describe "when not authorised at the project level", -> beforeEach () -> - @client.params.privilege_level = "unknown" + @client.ol_context.privilege_level = "unknown" it "should not allow access", () -> @AuthorizationManager.assertClientCanViewProjectAndDoc @client, @doc_id, (err) -> @@ -97,7 +89,7 @@ describe 'AuthorizationManager', -> describe "when authorised at the project level", -> beforeEach () -> - @client.params.privilege_level = "readOnly" + @client.ol_context.privilege_level = "readOnly" describe "and not authorised at the document level", -> it "should not allow access", () -> @@ -127,11 +119,11 @@ describe 'AuthorizationManager', -> beforeEach () -> @doc_id = "12345" @callback = sinon.stub() - @client.params = {} + @client.ol_context = {} describe "when not authorised at the project level", -> beforeEach () -> - @client.params.privilege_level = "readOnly" + @client.ol_context.privilege_level = "readOnly" it "should not allow access", () -> @AuthorizationManager.assertClientCanEditProjectAndDoc @client, @doc_id, (err) -> @@ -147,7 +139,7 @@ describe 'AuthorizationManager', -> describe "when authorised at the project level", -> beforeEach () -> - @client.params.privilege_level = "readAndWrite" + @client.ol_context.privilege_level = "readAndWrite" describe "and not authorised at the document level", -> it "should not allow access", () -> diff --git a/services/real-time/test/unit/coffee/WebsocketControllerTests.coffee b/services/real-time/test/unit/coffee/WebsocketControllerTests.coffee index 719a736f4a..c0047c49b7 100644 --- a/services/real-time/test/unit/coffee/WebsocketControllerTests.coffee +++ b/services/real-time/test/unit/coffee/WebsocketControllerTests.coffee @@ -23,9 +23,7 @@ describe 'WebsocketController', -> disconnected: false id: @client_id = "mock-client-id-123" publicId: "other-id-#{Math.random()}" - params: {} - set: sinon.stub() - get: (param, cb) -> cb null, @params[param] + ol_context: {} join: sinon.stub() leave: sinon.stub() @WebsocketController = SandboxedModule.require modulePath, requires: @@ -69,38 +67,27 @@ describe 'WebsocketController', -> @RoomManager.joinProject.calledWith(@client, @project_id).should.equal true it "should set the privilege level on the client", -> - @client.set.calledWith("privilege_level", @privilegeLevel).should.equal true - + @client.ol_context["privilege_level"].should.equal @privilegeLevel it "should set the user's id on the client", -> - @client.set.calledWith("user_id", @user._id).should.equal true - + @client.ol_context["user_id"].should.equal @user._id it "should set the user's email on the client", -> - @client.set.calledWith("email", @user.email).should.equal true - + @client.ol_context["email"].should.equal @user.email it "should set the user's first_name on the client", -> - @client.set.calledWith("first_name", @user.first_name).should.equal true - + @client.ol_context["first_name"].should.equal @user.first_name it "should set the user's last_name on the client", -> - @client.set.calledWith("last_name", @user.last_name).should.equal true - + @client.ol_context["last_name"].should.equal @user.last_name it "should set the user's sign up date on the client", -> - @client.set.calledWith("signup_date", @user.signUpDate).should.equal true - + @client.ol_context["signup_date"].should.equal @user.signUpDate it "should set the user's login_count on the client", -> - @client.set.calledWith("login_count", @user.loginCount).should.equal true - + @client.ol_context["login_count"].should.equal @user.loginCount it "should set the connected time on the client", -> - @client.set.calledWith("connected_time", new Date()).should.equal true - + @client.ol_context["connected_time"].should.equal new Date() it "should set the project_id on the client", -> - @client.set.calledWith("project_id", @project_id).should.equal true - + @client.ol_context["project_id"].should.equal @project_id it "should set the project owner id on the client", -> - @client.set.calledWith("owner_id", @owner_id).should.equal true - + @client.ol_context["owner_id"].should.equal @owner_id it "should set the is_restricted_user flag on the client", -> - @client.set.calledWith("is_restricted_user", @isRestrictedUser).should.equal true - + @client.ol_context["is_restricted_user"].should.equal @isRestrictedUser it "should call the callback with the project, privilegeLevel and protocolVersion", -> @callback .calledWith(null, @project, @privilegeLevel, @WebsocketController.PROTOCOL_VERSION) @@ -191,14 +178,14 @@ describe 'WebsocketController', -> if room_id != @project_id throw "expected room_id to be project_id" return @clientsInRoom - @client.params.project_id = @project_id - @client.params.user_id = @user_id + @client.ol_context.project_id = @project_id + @client.ol_context.user_id = @user_id @WebsocketController.FLUSH_IF_EMPTY_DELAY = 0 tk.reset() # Allow setTimeout to work. describe "when the client did not joined a project yet", -> beforeEach (done) -> - @client.params = {} + @client.ol_context = {} @WebsocketController.leaveProject @io, @client, done it "should bail out when calling leaveProject", () -> @@ -248,8 +235,8 @@ describe 'WebsocketController', -> describe "when client has not authenticated", -> beforeEach (done) -> - @client.params.user_id = null - @client.params.project_id = null + @client.ol_context.user_id = null + @client.ol_context.project_id = null @WebsocketController.leaveProject @io, @client, done it "should not end clientTracking.clientDisconnected to the project room", -> @@ -272,8 +259,8 @@ describe 'WebsocketController', -> describe "when client has not joined a project", -> beforeEach (done) -> - @client.params.user_id = @user_id - @client.params.project_id = null + @client.ol_context.user_id = @user_id + @client.ol_context.project_id = null @WebsocketController.leaveProject @io, @client, done it "should not end clientTracking.clientDisconnected to the project room", -> @@ -303,8 +290,8 @@ describe 'WebsocketController', -> @ranges = { "mock": "ranges" } @options = {} - @client.params.project_id = @project_id - @client.params.is_restricted_user = false + @client.ol_context.project_id = @project_id + @client.ol_context.is_restricted_user = false @AuthorizationManager.addAccessToDoc = sinon.stub() @AuthorizationManager.assertClientCanViewProject = sinon.stub().callsArgWith(1, null) @DocumentUpdaterManager.getDocument = sinon.stub().callsArgWith(3, null, @doc_lines, @version, @ranges, @ops) @@ -408,7 +395,7 @@ describe 'WebsocketController', -> describe "with a restricted client", -> beforeEach -> @ranges.comments = [{op: {a: 1}}, {op: {a: 2}}] - @client.params.is_restricted_user = true + @client.ol_context.is_restricted_user = true @WebsocketController.joinDoc @client, @doc_id, -1, @options, @callback it "should overwrite ranges.comments with an empty list", -> @@ -463,7 +450,7 @@ describe 'WebsocketController', -> describe "leaveDoc", -> beforeEach -> @doc_id = "doc-id-123" - @client.params.project_id = @project_id + @client.ol_context.project_id = @project_id @RoomManager.leaveDoc = sinon.stub() @WebsocketController.leaveDoc @client, @doc_id, @callback @@ -479,7 +466,7 @@ describe 'WebsocketController', -> describe "getConnectedUsers", -> beforeEach -> - @client.params.project_id = @project_id + @client.ol_context.project_id = @project_id @users = ["mock", "users"] @WebsocketLoadBalancer.emitToRoom = sinon.stub() @ConnectedUsersManager.getConnectedUsers = sinon.stub().callsArgWith(1, null, @users) @@ -527,7 +514,7 @@ describe 'WebsocketController', -> describe "when restricted user", -> beforeEach -> - @client.params.is_restricted_user = true + @client.ol_context.is_restricted_user = true @AuthorizationManager.assertClientCanViewProject = sinon.stub().callsArgWith(1, null) @WebsocketController.getConnectedUsers @client, @callback @@ -564,14 +551,13 @@ describe 'WebsocketController', -> describe "with a logged in user", -> beforeEach -> - @clientParams = { + @client.ol_context = { project_id: @project_id first_name: @first_name = "Douglas" last_name: @last_name = "Adams" email: @email = "joe@example.com" user_id: @user_id = "user-id-123" } - @client.get = (param, callback) => callback null, @clientParams[param] @WebsocketController.updateClientPosition @client, @update @populatedCursorData = @@ -604,14 +590,13 @@ describe 'WebsocketController', -> describe "with a logged in user who has no last_name set", -> beforeEach -> - @clientParams = { + @client.ol_context = { project_id: @project_id first_name: @first_name = "Douglas" last_name: undefined email: @email = "joe@example.com" user_id: @user_id = "user-id-123" } - @client.get = (param, callback) => callback null, @clientParams[param] @WebsocketController.updateClientPosition @client, @update @populatedCursorData = @@ -644,14 +629,13 @@ describe 'WebsocketController', -> describe "with a logged in user who has no first_name set", -> beforeEach -> - @clientParams = { + @client.ol_context = { project_id: @project_id first_name: undefined last_name: @last_name = "Adams" email: @email = "joe@example.com" user_id: @user_id = "user-id-123" } - @client.get = (param, callback) => callback null, @clientParams[param] @WebsocketController.updateClientPosition @client, @update @populatedCursorData = @@ -683,14 +667,13 @@ describe 'WebsocketController', -> @metrics.inc.calledWith("editor.update-client-position", 0.1).should.equal true describe "with a logged in user who has no names set", -> beforeEach -> - @clientParams = { + @client.ol_context = { project_id: @project_id first_name: undefined last_name: undefined email: @email = "joe@example.com" user_id: @user_id = "user-id-123" } - @client.get = (param, callback) => callback null, @clientParams[param] @WebsocketController.updateClientPosition @client, @update it "should send the update to the project name with no name", -> @@ -709,10 +692,9 @@ describe 'WebsocketController', -> describe "with an anonymous user", -> beforeEach -> - @clientParams = { + @client.ol_context = { project_id: @project_id } - @client.get = (param, callback) => callback null, @clientParams[param] @WebsocketController.updateClientPosition @client, @update it "should send the update to the project room with no name", -> @@ -745,8 +727,8 @@ describe 'WebsocketController', -> describe "applyOtUpdate", -> beforeEach -> @update = {op: {p: 12, t: "foo"}} - @client.params.user_id = @user_id - @client.params.project_id = @project_id + @client.ol_context.user_id = @user_id + @client.ol_context.project_id = @project_id @WebsocketController._assertClientCanApplyUpdate = sinon.stub().yields() @DocumentUpdaterManager.queueChange = sinon.stub().callsArg(3) @@ -807,8 +789,8 @@ describe 'WebsocketController', -> beforeEach (done) -> @client.disconnect = sinon.stub() @client.emit = sinon.stub() - @client.params.user_id = @user_id - @client.params.project_id = @project_id + @client.ol_context.user_id = @user_id + @client.ol_context.project_id = @project_id error = new Error("update is too large") error.updateSize = 7372835 @DocumentUpdaterManager.queueChange = sinon.stub().callsArgWith(3, error) diff --git a/services/real-time/test/unit/coffee/WebsocketLoadBalancerTests.coffee b/services/real-time/test/unit/coffee/WebsocketLoadBalancerTests.coffee index e6fe1df8c9..b2441cd6d0 100644 --- a/services/real-time/test/unit/coffee/WebsocketLoadBalancerTests.coffee +++ b/services/real-time/test/unit/coffee/WebsocketLoadBalancerTests.coffee @@ -18,12 +18,6 @@ describe "WebsocketLoadBalancer", -> "./RoomManager" : @RoomManager = {eventSource: sinon.stub().returns @RoomEvents} "./ChannelManager": @ChannelManager = {publish: sinon.stub()} "./ConnectedUsersManager": @ConnectedUsersManager = {refreshClient: sinon.stub()} - "./Utils": @Utils = { - getClientAttributes: sinon.spy( - (client, _attrs, callback) -> - callback(null, {is_restricted_user: !!client.__isRestricted}) - ) - } @io = {} @WebsocketLoadBalancer.rclientPubList = [{publish: sinon.stub()}] @WebsocketLoadBalancer.rclientSubList = [{ @@ -87,9 +81,9 @@ describe "WebsocketLoadBalancer", -> beforeEach -> @io.sockets = clients: sinon.stub().returns([ - {id: 'client-id-1', emit: @emit1 = sinon.stub()} - {id: 'client-id-2', emit: @emit2 = sinon.stub()} - {id: 'client-id-1', emit: @emit3 = sinon.stub()} # duplicate client + {id: 'client-id-1', emit: @emit1 = sinon.stub(), ol_context: {}} + {id: 'client-id-2', emit: @emit2 = sinon.stub(), ol_context: {}} + {id: 'client-id-1', emit: @emit3 = sinon.stub(), ol_context: {}} # duplicate client ]) data = JSON.stringify room_id: @room_id @@ -109,10 +103,10 @@ describe "WebsocketLoadBalancer", -> beforeEach -> @io.sockets = clients: sinon.stub().returns([ - {id: 'client-id-1', emit: @emit1 = sinon.stub()} - {id: 'client-id-2', emit: @emit2 = sinon.stub()} - {id: 'client-id-1', emit: @emit3 = sinon.stub()} # duplicate client - {id: 'client-id-4', emit: @emit4 = sinon.stub(), __isRestricted: true} + {id: 'client-id-1', emit: @emit1 = sinon.stub(), ol_context: {}} + {id: 'client-id-2', emit: @emit2 = sinon.stub(), ol_context: {}} + {id: 'client-id-1', emit: @emit3 = sinon.stub(), ol_context: {}} # duplicate client + {id: 'client-id-4', emit: @emit4 = sinon.stub(), ol_context: {is_restricted_user: true}} ]) data = JSON.stringify room_id: @room_id @@ -133,10 +127,10 @@ describe "WebsocketLoadBalancer", -> beforeEach -> @io.sockets = clients: sinon.stub().returns([ - {id: 'client-id-1', emit: @emit1 = sinon.stub()} - {id: 'client-id-2', emit: @emit2 = sinon.stub()} - {id: 'client-id-1', emit: @emit3 = sinon.stub()} # duplicate client - {id: 'client-id-4', emit: @emit4 = sinon.stub(), __isRestricted: true} + {id: 'client-id-1', emit: @emit1 = sinon.stub(), ol_context: {}} + {id: 'client-id-2', emit: @emit2 = sinon.stub(), ol_context: {}} + {id: 'client-id-1', emit: @emit3 = sinon.stub(), ol_context: {}} # duplicate client + {id: 'client-id-4', emit: @emit4 = sinon.stub(), ol_context: {is_restricted_user: true}} ]) data = JSON.stringify room_id: @room_id diff --git a/services/real-time/test/unit/coffee/helpers/MockClient.coffee b/services/real-time/test/unit/coffee/helpers/MockClient.coffee index 4dcddeb01d..497928132a 100644 --- a/services/real-time/test/unit/coffee/helpers/MockClient.coffee +++ b/services/real-time/test/unit/coffee/helpers/MockClient.coffee @@ -4,15 +4,10 @@ idCounter = 0 module.exports = class MockClient constructor: () -> - @attributes = {} + @ol_context = {} @join = sinon.stub() @emit = sinon.stub() @disconnect = sinon.stub() @id = idCounter++ @publicId = idCounter++ - set : (key, value, callback) -> - @attributes[key] = value - callback() if callback? - get : (key, callback) -> - callback null, @attributes[key] disconnect: () -> From a70c1e1fa2aa3e607f997f711b5f9893266f49b6 Mon Sep 17 00:00:00 2001 From: Jakob Ackermann Date: Mon, 24 Feb 2020 14:39:34 +0100 Subject: [PATCH 360/491] [perf] WebsocketLoadBalancer: check is_restricted_message once ...and filter clients early on. --- .../real-time/app/coffee/WebsocketLoadBalancer.coffee | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/services/real-time/app/coffee/WebsocketLoadBalancer.coffee b/services/real-time/app/coffee/WebsocketLoadBalancer.coffee index 5a59ea504f..17de2df2b6 100644 --- a/services/real-time/app/coffee/WebsocketLoadBalancer.coffee +++ b/services/real-time/app/coffee/WebsocketLoadBalancer.coffee @@ -77,8 +77,15 @@ module.exports = WebsocketLoadBalancer = status = EventLogger.checkEventOrder("editor-events", message._id, message) if status is "duplicate" return # skip duplicate events + + is_restricted_message = message.message not in RESTRICTED_USER_MESSAGE_TYPE_PASS_LIST + # send messages only to unique clients (due to duplicate entries in io.sockets.clients) clientList = io.sockets.clients(message.room_id) + .filter((client) -> + !(is_restricted_message && client.ol_context['is_restricted_user']) + ) + # avoid unnecessary work if no clients are connected return if clientList.length is 0 logger.log { @@ -93,11 +100,9 @@ module.exports = WebsocketLoadBalancer = Async.eachLimit clientList , 2 , (client, cb) -> - is_restricted_user = client.ol_context['is_restricted_user'] if !seen[client.id] seen[client.id] = true - if !(is_restricted_user && message.message not in RESTRICTED_USER_MESSAGE_TYPE_PASS_LIST) - client.emit(message.message, message.payload...) + client.emit(message.message, message.payload...) cb() , (err) -> if err? From ce4f9148c3f1ac02059cc808490588ecef29644e Mon Sep 17 00:00:00 2001 From: Jakob Ackermann Date: Wed, 26 Feb 2020 10:57:14 +0100 Subject: [PATCH 361/491] [perf] WebsocketLoadBalancer: move back to a sync loop for msg fan out --- .../real-time/app/coffee/WebsocketLoadBalancer.coffee | 10 +--------- 1 file changed, 1 insertion(+), 9 deletions(-) diff --git a/services/real-time/app/coffee/WebsocketLoadBalancer.coffee b/services/real-time/app/coffee/WebsocketLoadBalancer.coffee index 17de2df2b6..209ec0bb08 100644 --- a/services/real-time/app/coffee/WebsocketLoadBalancer.coffee +++ b/services/real-time/app/coffee/WebsocketLoadBalancer.coffee @@ -7,7 +7,6 @@ HealthCheckManager = require "./HealthCheckManager" RoomManager = require "./RoomManager" ChannelManager = require "./ChannelManager" ConnectedUsersManager = require "./ConnectedUsersManager" -Async = require 'async' RESTRICTED_USER_MESSAGE_TYPE_PASS_LIST = [ 'connectionAccepted', @@ -96,17 +95,10 @@ module.exports = WebsocketLoadBalancer = socketIoClients: (client.id for client in clientList) }, "distributing event to clients" seen = {} - # Send the messages to clients async, don't wait for them all to finish - Async.eachLimit clientList - , 2 - , (client, cb) -> + for client in clientList if !seen[client.id] seen[client.id] = true client.emit(message.message, message.payload...) - cb() - , (err) -> - if err? - logger.err {err, message}, "Error sending message to clients" else if message.health_check? logger.debug {message}, "got health check message in editor events channel" HealthCheckManager.check channel, message.key From d17ef183d01997c418e57cc95b5633575e91dd5b Mon Sep 17 00:00:00 2001 From: Jakob Ackermann Date: Tue, 21 Apr 2020 12:45:23 +0100 Subject: [PATCH 362/491] [Router] gracefully set and do not reset the ol_context --- services/real-time/app/coffee/Router.coffee | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/services/real-time/app/coffee/Router.coffee b/services/real-time/app/coffee/Router.coffee index f89e1d9eaf..b7d1ae2662 100644 --- a/services/real-time/app/coffee/Router.coffee +++ b/services/real-time/app/coffee/Router.coffee @@ -55,7 +55,7 @@ module.exports = Router = app.post "/client/:client_id/disconnect", httpAuth, HttpApiController.disconnectClient session.on 'connection', (error, client, session) -> - client.ol_context = {} + client.ol_context = {} unless client.ol_context client?.on "error", (err) -> logger.err { clientErr: err }, "socket.io client error" @@ -113,13 +113,9 @@ module.exports = Router = metrics.inc('socket-io.disconnect') metrics.gauge('socket-io.clients', io.sockets.clients()?.length - 1) - cleanup = () -> - delete client.ol_context WebsocketController.leaveProject io, client, (err) -> if err? - Router._handleError cleanup, err, client, "leaveProject" - else - cleanup() + Router._handleError (() ->), err, client, "leaveProject" # Variadic. The possible arguments: # doc_id, callback From 1fcf534dcfcb161c92321b2f4730a2921a4af934 Mon Sep 17 00:00:00 2001 From: Jakob Ackermann Date: Wed, 22 Apr 2020 11:36:20 +0200 Subject: [PATCH 363/491] [Router] revert preserving of client.ol_context We do not enter this line twice, it would result in multiple event handlers too. --- services/real-time/app/coffee/Router.coffee | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/services/real-time/app/coffee/Router.coffee b/services/real-time/app/coffee/Router.coffee index b7d1ae2662..3d891f1476 100644 --- a/services/real-time/app/coffee/Router.coffee +++ b/services/real-time/app/coffee/Router.coffee @@ -55,7 +55,9 @@ module.exports = Router = app.post "/client/:client_id/disconnect", httpAuth, HttpApiController.disconnectClient session.on 'connection', (error, client, session) -> - client.ol_context = {} unless client.ol_context + # init client context, we may access it in Router._handleError before + # setting any values + client.ol_context = {} client?.on "error", (err) -> logger.err { clientErr: err }, "socket.io client error" From 205efa8812d7ecd5ea49c49f8dda77819ba3c13a Mon Sep 17 00:00:00 2001 From: Jakob Ackermann Date: Wed, 22 Apr 2020 11:37:16 +0200 Subject: [PATCH 364/491] [WebsocketController] reset the client context before (re)populating it Co-Authored-By: Brian Gough --- services/real-time/app/coffee/WebsocketController.coffee | 1 + 1 file changed, 1 insertion(+) diff --git a/services/real-time/app/coffee/WebsocketController.coffee b/services/real-time/app/coffee/WebsocketController.coffee index 3ca030ca0c..f93e67f2a2 100644 --- a/services/real-time/app/coffee/WebsocketController.coffee +++ b/services/real-time/app/coffee/WebsocketController.coffee @@ -33,6 +33,7 @@ module.exports = WebsocketController = logger.warn {err, project_id, user_id, client_id: client.id}, "user is not authorized to join project" return callback(err) + client.ol_context = {} client.ol_context["privilege_level"] = privilegeLevel client.ol_context["user_id"] = user_id client.ol_context["project_id"] = project_id From 8ac973e76782db724824c52a00e8ab658bc4f56e Mon Sep 17 00:00:00 2001 From: Jakob Ackermann Date: Tue, 23 Jun 2020 17:02:51 +0100 Subject: [PATCH 365/491] [misc] help prettier/decaffeinate with scoping a comment into a fn body --- .../real-time/test/acceptance/coffee/DrainManagerTests.coffee | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/services/real-time/test/acceptance/coffee/DrainManagerTests.coffee b/services/real-time/test/acceptance/coffee/DrainManagerTests.coffee index b5b192cf88..ca967408d8 100644 --- a/services/real-time/test/acceptance/coffee/DrainManagerTests.coffee +++ b/services/real-time/test/acceptance/coffee/DrainManagerTests.coffee @@ -71,8 +71,7 @@ describe "DrainManagerTests", -> ], done afterEach (done) -> - # reset drain - drain(0, done) + drain(0, done) # reset drain it "should not timeout", -> expect(true).to.equal(true) From 59083edb9e045c75eb852692efe81dbb9c8515e5 Mon Sep 17 00:00:00 2001 From: decaffeinate Date: Tue, 23 Jun 2020 18:29:17 +0100 Subject: [PATCH 366/491] decaffeinate: update build scripts to es --- services/real-time/.dockerignore | 2 - services/real-time/.eslintrc | 64 + services/real-time/.prettierrc | 7 + services/real-time/Dockerfile | 4 +- services/real-time/Jenkinsfile | 7 + services/real-time/Makefile | 61 +- services/real-time/buildscript.txt | 10 +- services/real-time/docker-compose.ci.yml | 1 - services/real-time/docker-compose.yml | 1 - services/real-time/nodemon.json | 7 +- services/real-time/package-lock.json | 2773 +++++++++++++++++++++- services/real-time/package.json | 37 +- 12 files changed, 2906 insertions(+), 68 deletions(-) create mode 100644 services/real-time/.eslintrc create mode 100644 services/real-time/.prettierrc diff --git a/services/real-time/.dockerignore b/services/real-time/.dockerignore index 386f26df30..ba1c3442de 100644 --- a/services/real-time/.dockerignore +++ b/services/real-time/.dockerignore @@ -5,5 +5,3 @@ gitrev .npm .nvmrc nodemon.json -app.js -**/js/* diff --git a/services/real-time/.eslintrc b/services/real-time/.eslintrc new file mode 100644 index 0000000000..76dad1561d --- /dev/null +++ b/services/real-time/.eslintrc @@ -0,0 +1,64 @@ +// this file was auto-generated, do not edit it directly. +// instead run bin/update_build_scripts from +// https://github.com/sharelatex/sharelatex-dev-environment +{ + "extends": [ + "standard", + "prettier", + "prettier/standard" + ], + "parserOptions": { + "ecmaVersion": 2018 + }, + "plugins": [ + "mocha", + "chai-expect", + "chai-friendly" + ], + "env": { + "node": true, + "mocha": true + }, + "rules": { + // Swap the no-unused-expressions rule with a more chai-friendly one + "no-unused-expressions": 0, + "chai-friendly/no-unused-expressions": "error" + }, + "overrides": [ + { + // Test specific rules + "files": ["test/**/*.js"], + "globals": { + "expect": true + }, + "rules": { + // mocha-specific rules + "mocha/handle-done-callback": "error", + "mocha/no-exclusive-tests": "error", + "mocha/no-global-tests": "error", + "mocha/no-identical-title": "error", + "mocha/no-nested-tests": "error", + "mocha/no-pending-tests": "error", + "mocha/no-skipped-tests": "error", + "mocha/no-mocha-arrows": "error", + + // chai-specific rules + "chai-expect/missing-assertion": "error", + "chai-expect/terminating-properties": "error", + + // prefer-arrow-callback applies to all callbacks, not just ones in mocha tests. + // we don't enforce this at the top-level - just in tests to manage `this` scope + // based on mocha's context mechanism + "mocha/prefer-arrow-callback": "error" + } + }, + { + // Backend specific rules + "files": ["app/**/*.js", "app.js", "index.js"], + "rules": { + // don't allow console.log in backend code + "no-console": "error" + } + } + ] +} diff --git a/services/real-time/.prettierrc b/services/real-time/.prettierrc new file mode 100644 index 0000000000..24f9ec526f --- /dev/null +++ b/services/real-time/.prettierrc @@ -0,0 +1,7 @@ +# This file was auto-generated, do not edit it directly. +# Instead run bin/update_build_scripts from +# https://github.com/sharelatex/sharelatex-dev-environment +{ + "semi": false, + "singleQuote": true +} diff --git a/services/real-time/Dockerfile b/services/real-time/Dockerfile index 71e74fe251..b07f7117bc 100644 --- a/services/real-time/Dockerfile +++ b/services/real-time/Dockerfile @@ -1,7 +1,6 @@ # This file was auto-generated, do not edit it directly. # Instead run bin/update_build_scripts from # https://github.com/sharelatex/sharelatex-dev-environment -# Version: 1.3.5 FROM node:10.21.0 as base @@ -12,12 +11,11 @@ FROM base as app #wildcard as some files may not be in all repos COPY package*.json npm-shrink*.json /app/ -RUN npm install --quiet +RUN npm ci --quiet COPY . /app -RUN npm run compile:all FROM base diff --git a/services/real-time/Jenkinsfile b/services/real-time/Jenkinsfile index 684fb7daca..4fc4f79e8a 100644 --- a/services/real-time/Jenkinsfile +++ b/services/real-time/Jenkinsfile @@ -37,6 +37,13 @@ pipeline { } } + stage('Linting') { + steps { + sh 'DOCKER_COMPOSE_FLAGS="-f docker-compose.ci.yml" make format' + sh 'DOCKER_COMPOSE_FLAGS="-f docker-compose.ci.yml" make lint' + } + } + stage('Unit Tests') { steps { sh 'DOCKER_COMPOSE_FLAGS="-f docker-compose.ci.yml" make test_unit' diff --git a/services/real-time/Makefile b/services/real-time/Makefile index d8b68a699f..437700ee2f 100644 --- a/services/real-time/Makefile +++ b/services/real-time/Makefile @@ -1,11 +1,12 @@ # This file was auto-generated, do not edit it directly. # Instead run bin/update_build_scripts from # https://github.com/sharelatex/sharelatex-dev-environment -# Version: 1.3.5 BUILD_NUMBER ?= local BRANCH_NAME ?= $(shell git rev-parse --abbrev-ref HEAD) PROJECT_NAME = real-time +BUILD_DIR_NAME = $(shell pwd | xargs basename | tr -cd '[a-zA-Z0-9_.\-]') + DOCKER_COMPOSE_FLAGS ?= -f docker-compose.yml DOCKER_COMPOSE := BUILD_NUMBER=$(BUILD_NUMBER) \ BRANCH_NAME=$(BRANCH_NAME) \ @@ -13,34 +14,63 @@ DOCKER_COMPOSE := BUILD_NUMBER=$(BUILD_NUMBER) \ MOCHA_GREP=${MOCHA_GREP} \ docker-compose ${DOCKER_COMPOSE_FLAGS} +DOCKER_COMPOSE_TEST_ACCEPTANCE = \ + COMPOSE_PROJECT_NAME=test_acceptance_$(BUILD_DIR_NAME) $(DOCKER_COMPOSE) + +DOCKER_COMPOSE_TEST_UNIT = \ + COMPOSE_PROJECT_NAME=test_unit_$(BUILD_DIR_NAME) $(DOCKER_COMPOSE) + clean: docker rmi ci/$(PROJECT_NAME):$(BRANCH_NAME)-$(BUILD_NUMBER) docker rmi gcr.io/overleaf-ops/$(PROJECT_NAME):$(BRANCH_NAME)-$(BUILD_NUMBER) - rm -f app.js - rm -rf app/js - rm -rf test/unit/js - rm -rf test/acceptance/js -test: test_unit test_acceptance +format: + $(DOCKER_COMPOSE) run --rm test_unit npm run format + +format_fix: + $(DOCKER_COMPOSE) run --rm test_unit npm run format:fix + +lint: + $(DOCKER_COMPOSE) run --rm test_unit npm run lint + +test: format lint test_unit test_acceptance test_unit: - @[ ! -d test/unit ] && echo "real-time has no unit tests" || $(DOCKER_COMPOSE) run --rm test_unit +ifneq (,$(wildcard test/unit)) + $(DOCKER_COMPOSE_TEST_UNIT) run --rm test_unit + $(MAKE) test_unit_clean +endif -test_acceptance: test_clean test_acceptance_pre_run test_acceptance_run +test_clean: test_unit_clean +test_unit_clean: +ifneq (,$(wildcard test/unit)) + $(DOCKER_COMPOSE_TEST_UNIT) down -v -t 0 +endif -test_acceptance_debug: test_clean test_acceptance_pre_run test_acceptance_run_debug +test_acceptance: test_acceptance_clean test_acceptance_pre_run test_acceptance_run + $(MAKE) test_acceptance_clean + +test_acceptance_debug: test_acceptance_clean test_acceptance_pre_run test_acceptance_run_debug + $(MAKE) test_acceptance_clean test_acceptance_run: - @[ ! -d test/acceptance ] && echo "real-time has no acceptance tests" || $(DOCKER_COMPOSE) run --rm test_acceptance +ifneq (,$(wildcard test/acceptance)) + $(DOCKER_COMPOSE_TEST_ACCEPTANCE) run --rm test_acceptance +endif test_acceptance_run_debug: - @[ ! -d test/acceptance ] && echo "real-time has no acceptance tests" || $(DOCKER_COMPOSE) run -p 127.0.0.9:19999:19999 --rm test_acceptance npm run test:acceptance -- --inspect=0.0.0.0:19999 --inspect-brk +ifneq (,$(wildcard test/acceptance)) + $(DOCKER_COMPOSE_TEST_ACCEPTANCE) run -p 127.0.0.9:19999:19999 --rm test_acceptance npm run test:acceptance -- --inspect=0.0.0.0:19999 --inspect-brk +endif -test_clean: - $(DOCKER_COMPOSE) down -v -t 0 +test_clean: test_acceptance_clean +test_acceptance_clean: + $(DOCKER_COMPOSE_TEST_ACCEPTANCE) down -v -t 0 test_acceptance_pre_run: - @[ ! -f test/acceptance/js/scripts/pre-run ] && echo "real-time has no pre acceptance tests task" || $(DOCKER_COMPOSE) run --rm test_acceptance test/acceptance/js/scripts/pre-run +ifneq (,$(wildcard test/acceptance/js/scripts/pre-run)) + $(DOCKER_COMPOSE_TEST_ACCEPTANCE) run --rm test_acceptance test/acceptance/js/scripts/pre-run +endif build: docker build --pull --tag ci/$(PROJECT_NAME):$(BRANCH_NAME)-$(BUILD_NUMBER) \ @@ -54,8 +84,5 @@ publish: docker push $(DOCKER_REPO)/$(PROJECT_NAME):$(BRANCH_NAME)-$(BUILD_NUMBER) -lint: - -format: .PHONY: clean test test_unit test_acceptance test_clean build publish diff --git a/services/real-time/buildscript.txt b/services/real-time/buildscript.txt index 7e1b0350d4..526bd01e14 100644 --- a/services/real-time/buildscript.txt +++ b/services/real-time/buildscript.txt @@ -1,10 +1,10 @@ real-time ---public-repo=True ---language=coffeescript ---env-add= ---node-version=10.21.0 --acceptance-creds=None --dependencies=redis --docker-repos=gcr.io/overleaf-ops +--env-add= --env-pass-through= ---script-version=1.3.5 +--language=es +--node-version=10.21.0 +--public-repo=True +--script-version=2.3.0 diff --git a/services/real-time/docker-compose.ci.yml b/services/real-time/docker-compose.ci.yml index 292c297cc3..f9fc7b983e 100644 --- a/services/real-time/docker-compose.ci.yml +++ b/services/real-time/docker-compose.ci.yml @@ -1,7 +1,6 @@ # This file was auto-generated, do not edit it directly. # Instead run bin/update_build_scripts from # https://github.com/sharelatex/sharelatex-dev-environment -# Version: 1.3.5 version: "2.3" diff --git a/services/real-time/docker-compose.yml b/services/real-time/docker-compose.yml index b174363e67..8570f506ae 100644 --- a/services/real-time/docker-compose.yml +++ b/services/real-time/docker-compose.yml @@ -1,7 +1,6 @@ # This file was auto-generated, do not edit it directly. # Instead run bin/update_build_scripts from # https://github.com/sharelatex/sharelatex-dev-environment -# Version: 1.3.5 version: "2.3" diff --git a/services/real-time/nodemon.json b/services/real-time/nodemon.json index 98db38d71b..5826281b84 100644 --- a/services/real-time/nodemon.json +++ b/services/real-time/nodemon.json @@ -10,10 +10,9 @@ }, "watch": [ - "app/coffee/", - "app.coffee", + "app/js/", + "app.js", "config/" ], - "ext": "coffee" - + "ext": "js" } diff --git a/services/real-time/package-lock.json b/services/real-time/package-lock.json index b6b2a7a811..616f0edea9 100644 --- a/services/real-time/package-lock.json +++ b/services/real-time/package-lock.json @@ -4,6 +4,32 @@ "lockfileVersion": 1, "requires": true, "dependencies": { + "@babel/code-frame": { + "version": "7.10.3", + "resolved": "https://registry.npmjs.org/@babel/code-frame/-/code-frame-7.10.3.tgz", + "integrity": "sha512-fDx9eNW0qz0WkUeqL6tXEXzVlPh6Y5aCDEZesl0xBGA8ndRukX91Uk44ZqnkECp01NAZUdCAl+aiQNGi0k88Eg==", + "dev": true, + "requires": { + "@babel/highlight": "^7.10.3" + } + }, + "@babel/helper-validator-identifier": { + "version": "7.10.3", + "resolved": "https://registry.npmjs.org/@babel/helper-validator-identifier/-/helper-validator-identifier-7.10.3.tgz", + "integrity": "sha512-bU8JvtlYpJSBPuj1VUmKpFGaDZuLxASky3LhaKj3bmpSTY6VWooSM8msk+Z0CZoErFye2tlABF6yDkT3FOPAXw==", + "dev": true + }, + "@babel/highlight": { + "version": "7.10.3", + "resolved": "https://registry.npmjs.org/@babel/highlight/-/highlight-7.10.3.tgz", + "integrity": "sha512-Ih9B/u7AtgEnySE2L2F0Xm0GaM729XqqLfHkalTsbjXGyqmf/6M0Cu0WpvqueUlW+xk88BHw9Nkpj49naU+vWw==", + "dev": true, + "requires": { + "@babel/helper-validator-identifier": "^7.10.3", + "chalk": "^2.0.0", + "js-tokens": "^4.0.0" + } + }, "@google-cloud/common": { "version": "0.32.1", "resolved": "https://registry.npmjs.org/@google-cloud/common/-/common-0.32.1.tgz", @@ -244,6 +270,12 @@ "resolved": "https://registry.npmjs.org/@types/caseless/-/caseless-0.12.2.tgz", "integrity": "sha512-6ckxMjBBD8URvjB6J3NcnuAn5Pkl7t3TizAg+xdlzzQGSPSmBcXf8KoIH0ua/i+tio+ZRUHEXp0HEmvaR4kt0w==" }, + "@types/color-name": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/@types/color-name/-/color-name-1.1.1.tgz", + "integrity": "sha512-rr+OQyAjxze7GgWrSaJwydHStIhHq2lvY3BOC2Mj7KnzI7XK0Uw1TOOdI9lDoajEbSWLiYgoo4f1R51erQfhPQ==", + "dev": true + }, "@types/console-log-level": { "version": "1.4.0", "resolved": "https://registry.npmjs.org/@types/console-log-level/-/console-log-level-1.4.0.tgz", @@ -257,6 +289,24 @@ "@types/node": "*" } }, + "@types/eslint-visitor-keys": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/@types/eslint-visitor-keys/-/eslint-visitor-keys-1.0.0.tgz", + "integrity": "sha512-OCutwjDZ4aFS6PB1UZ988C4YgwlBHJd6wCeQqaLdmadZ/7e+w79+hbMUFC1QXDNCmdyoRfAFdm0RypzwR+Qpag==", + "dev": true + }, + "@types/json-schema": { + "version": "7.0.5", + "resolved": "https://registry.npmjs.org/@types/json-schema/-/json-schema-7.0.5.tgz", + "integrity": "sha512-7+2BITlgjgDhH0vvwZU/HZJVyk+2XUlvxXe8dFMedNX/aMkaOq++rMAFXc0tM7ij15QaWlbdQASBR9dihi+bDQ==", + "dev": true + }, + "@types/json5": { + "version": "0.0.29", + "resolved": "https://registry.npmjs.org/@types/json5/-/json5-0.0.29.tgz", + "integrity": "sha512-dRLjCWHYg4oaA77cxO64oO+7JwCwnIzkZPdrrC71jQmQtlhM556pwKo5bUzqvZndkVbeFLIIi+9TC40JNF5hNQ==", + "dev": true + }, "@types/long": { "version": "4.0.1", "resolved": "https://registry.npmjs.org/@types/long/-/long-4.0.1.tgz", @@ -300,6 +350,59 @@ "resolved": "https://registry.npmjs.org/@types/tough-cookie/-/tough-cookie-2.3.6.tgz", "integrity": "sha512-wHNBMnkoEBiRAd3s8KTKwIuO9biFtTf0LehITzBhSco+HQI0xkXZbLOD55SW3Aqw3oUkHstkm5SPv58yaAdFPQ==" }, + "@typescript-eslint/experimental-utils": { + "version": "1.13.0", + "resolved": "https://registry.npmjs.org/@typescript-eslint/experimental-utils/-/experimental-utils-1.13.0.tgz", + "integrity": "sha512-zmpS6SyqG4ZF64ffaJ6uah6tWWWgZ8m+c54XXgwFtUv0jNz8aJAVx8chMCvnk7yl6xwn8d+d96+tWp7fXzTuDg==", + "dev": true, + "requires": { + "@types/json-schema": "^7.0.3", + "@typescript-eslint/typescript-estree": "1.13.0", + "eslint-scope": "^4.0.0" + }, + "dependencies": { + "eslint-scope": { + "version": "4.0.3", + "resolved": "https://registry.npmjs.org/eslint-scope/-/eslint-scope-4.0.3.tgz", + "integrity": "sha512-p7VutNr1O/QrxysMo3E45FjYDTeXBy0iTltPFNSqKAIfjDSXC+4dj+qfyuD8bfAXrW/y6lW3O76VaYNPKfpKrg==", + "dev": true, + "requires": { + "esrecurse": "^4.1.0", + "estraverse": "^4.1.1" + } + } + } + }, + "@typescript-eslint/parser": { + "version": "1.13.0", + "resolved": "https://registry.npmjs.org/@typescript-eslint/parser/-/parser-1.13.0.tgz", + "integrity": "sha512-ITMBs52PCPgLb2nGPoeT4iU3HdQZHcPaZVw+7CsFagRJHUhyeTgorEwHXhFf3e7Evzi8oujKNpHc8TONth8AdQ==", + "dev": true, + "requires": { + "@types/eslint-visitor-keys": "^1.0.0", + "@typescript-eslint/experimental-utils": "1.13.0", + "@typescript-eslint/typescript-estree": "1.13.0", + "eslint-visitor-keys": "^1.0.0" + } + }, + "@typescript-eslint/typescript-estree": { + "version": "1.13.0", + "resolved": "https://registry.npmjs.org/@typescript-eslint/typescript-estree/-/typescript-estree-1.13.0.tgz", + "integrity": "sha512-b5rCmd2e6DCC6tCTN9GSUAuxdYwCM/k/2wdjHGrIRGPSJotWMCe/dGpi66u42bhuh8q3QBzqM4TMA1GUUCJvdw==", + "dev": true, + "requires": { + "lodash.unescape": "4.0.1", + "semver": "5.5.0" + }, + "dependencies": { + "semver": { + "version": "5.5.0", + "resolved": "https://registry.npmjs.org/semver/-/semver-5.5.0.tgz", + "integrity": "sha512-4SJ3dm0WAwWy/NVeioZh5AntkdJoWKxHxcmyP622fOkgHa4z3R0TdBJICINyaSDE6uNwVc8gZr+ZinwZAH4xIA==", + "dev": true + } + } + }, "abort-controller": { "version": "3.0.0", "resolved": "https://registry.npmjs.org/abort-controller/-/abort-controller-3.0.0.tgz", @@ -337,6 +440,12 @@ "resolved": "https://registry.npmjs.org/acorn/-/acorn-6.4.1.tgz", "integrity": "sha512-ZVA9k326Nwrj3Cj9jlh3wGFutC2ZornPNARZwsNYqQYgN0EsV2d53w5RN/co65Ohn4sUAUtb1rSUAOD6XN9idA==" }, + "acorn-jsx": { + "version": "5.2.0", + "resolved": "https://registry.npmjs.org/acorn-jsx/-/acorn-jsx-5.2.0.tgz", + "integrity": "sha512-HiUX/+K2YpkpJ+SzBffkM/AQ2YE03S0U1kjTLVpoJdhZMOWy8qvXVN9JdLqv2QsaQ6MPYQIuNmwD8zOiYUofLQ==", + "dev": true + }, "active-x-obfuscator": { "version": "0.0.1", "resolved": "https://registry.npmjs.org/active-x-obfuscator/-/active-x-obfuscator-0.0.1.tgz", @@ -364,11 +473,73 @@ "uri-js": "^4.2.2" } }, + "ansi-escapes": { + "version": "4.3.1", + "resolved": "https://registry.npmjs.org/ansi-escapes/-/ansi-escapes-4.3.1.tgz", + "integrity": "sha512-JWF7ocqNrp8u9oqpgV+wH5ftbt+cfvv+PTjOvKLT3AdYly/LmORARfEVT1iyjwN+4MqE5UmVKoAdIBqeoCHgLA==", + "dev": true, + "requires": { + "type-fest": "^0.11.0" + }, + "dependencies": { + "type-fest": { + "version": "0.11.0", + "resolved": "https://registry.npmjs.org/type-fest/-/type-fest-0.11.0.tgz", + "integrity": "sha512-OdjXJxnCN1AvyLSzeKIgXTXxV+99ZuXl3Hpo9XpJAv9MBcHrrJOQ5kV7ypXOuQie+AmWG25hLbiKdwYTifzcfQ==", + "dev": true + } + } + }, + "ansi-regex": { + "version": "5.0.0", + "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-5.0.0.tgz", + "integrity": "sha512-bY6fj56OUQ0hU1KjFNDQuJFezqKdrAyFdIevADiqrWHwSlbmBNMHp5ak2f40Pm8JTFyM2mqxkG6ngkHO11f/lg==", + "dev": true + }, + "ansi-styles": { + "version": "3.2.1", + "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-3.2.1.tgz", + "integrity": "sha512-VT0ZI6kZRdTh8YyJw3SMbYm/u+NqfsAxEpWO0Pf9sq8/e94WxxOpPKx9FR1FlyCtOVDNOQ+8ntlqFxiRc+r5qA==", + "dev": true, + "requires": { + "color-convert": "^1.9.0" + } + }, + "argparse": { + "version": "1.0.10", + "resolved": "https://registry.npmjs.org/argparse/-/argparse-1.0.10.tgz", + "integrity": "sha512-o5Roy6tNG4SL/FOkCAN6RzjiakZS25RLYFrcMttJqbdd8BWrnA+fGz57iN5Pb06pvBGvl5gQ0B48dJlslXvoTg==", + "dev": true, + "requires": { + "sprintf-js": "~1.0.2" + } + }, "array-flatten": { "version": "1.1.1", "resolved": "https://registry.npmjs.org/array-flatten/-/array-flatten-1.1.1.tgz", "integrity": "sha512-PCVAQswWemu6UdxsDFFX/+gVeYqKAod3D3UVm91jHwynguOwAvYPhx8nNlM++NqRcK6CxxpUafjmhIdKiHibqg==" }, + "array-includes": { + "version": "3.1.1", + "resolved": "https://registry.npmjs.org/array-includes/-/array-includes-3.1.1.tgz", + "integrity": "sha512-c2VXaCHl7zPsvpkFsw4nxvFie4fh1ur9bpcgsVkIjqn0H/Xwdg+7fv3n2r/isyS8EBj5b06M9kHyZuIr4El6WQ==", + "dev": true, + "requires": { + "define-properties": "^1.1.3", + "es-abstract": "^1.17.0", + "is-string": "^1.0.5" + } + }, + "array.prototype.flat": { + "version": "1.2.3", + "resolved": "https://registry.npmjs.org/array.prototype.flat/-/array.prototype.flat-1.2.3.tgz", + "integrity": "sha512-gBlRZV0VSmfPIeWfuuy56XZMvbVfbEUnOXUvt3F/eUUUSyzlgLxhEX4YAEpxNAogRGehPSnfXyPtYyKAhkzQhQ==", + "dev": true, + "requires": { + "define-properties": "^1.1.3", + "es-abstract": "^1.17.0-next.1" + } + }, "arrify": { "version": "2.0.1", "resolved": "https://registry.npmjs.org/arrify/-/arrify-2.0.1.tgz", @@ -393,6 +564,12 @@ "integrity": "sha512-g/gZV+G476cnmtYI+Ko9d5khxSoCSoom/EaNmmCfwpOvBXEJ18qwFrxfP1/CsIqk2no1sAKKwxndV0tP7ROOFQ==", "dev": true }, + "astral-regex": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/astral-regex/-/astral-regex-1.0.0.tgz", + "integrity": "sha512-+Ryf6g3BKoRc7jfp7ad8tM4TtMiaWvbF/1/sQcZPkkS7ag3D5nMBCe2UfOTONtAkaG0tO0ij3C5Lwmf1EiyjHg==", + "dev": true + }, "async": { "version": "0.9.2", "resolved": "https://registry.npmjs.org/async/-/async-0.9.2.tgz", @@ -501,6 +678,12 @@ "type-is": "~1.6.17" } }, + "boolify": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/boolify/-/boolify-1.0.1.tgz", + "integrity": "sha512-ma2q0Tc760dW54CdOyJjhrg/a54317o1zYADQJFgperNGKIKgAUGIcKnuMiff8z57+yGlrGNEt4lPgZfCgTJgA==", + "dev": true + }, "brace-expansion": { "version": "1.1.11", "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-1.1.11.tgz", @@ -541,6 +724,29 @@ "resolved": "https://registry.npmjs.org/bytes/-/bytes-3.1.0.tgz", "integrity": "sha512-zauLjrfCG+xvoyaqLoV8bLVXXNGC4JqlxFCutSDWA6fJrTo2ZuvLYTqZ7aHBLZSMOopbzwv8f+wZcVzfVTI2Dg==" }, + "callsites": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/callsites/-/callsites-3.1.0.tgz", + "integrity": "sha512-P8BjAsXvZS+VIDUI11hHCQEv74YT67YUi5JJFNWIqL235sBmjX4+qx9Muvls5ivyNENctx46xQLQ3aTuE7ssaQ==", + "dev": true + }, + "camelcase": { + "version": "5.3.1", + "resolved": "https://registry.npmjs.org/camelcase/-/camelcase-5.3.1.tgz", + "integrity": "sha512-L28STB170nwWS63UjtlEOE3dldQApaJXZkOI1uMFfzf3rRuPegHaHesyee+YxQ+W6SvRDQV6UrdOdRiR153wJg==", + "dev": true + }, + "camelcase-keys": { + "version": "6.2.2", + "resolved": "https://registry.npmjs.org/camelcase-keys/-/camelcase-keys-6.2.2.tgz", + "integrity": "sha512-YrwaA0vEKazPBkn0ipTiMpSajYDSe+KjQfrjhcBMxJt/znbvlHd8Pw/Vamaz5EB4Wfhs3SUR3Z9mwRu/P3s3Yg==", + "dev": true, + "requires": { + "camelcase": "^5.3.1", + "map-obj": "^4.0.0", + "quick-lru": "^4.0.1" + } + }, "caseless": { "version": "0.12.0", "resolved": "https://registry.npmjs.org/caseless/-/caseless-0.12.0.tgz", @@ -556,6 +762,74 @@ "deep-eql": "0.1.3" } }, + "chalk": { + "version": "2.4.2", + "resolved": "https://registry.npmjs.org/chalk/-/chalk-2.4.2.tgz", + "integrity": "sha512-Mti+f9lpJNcwF4tWV8/OrTTtF1gZi+f8FqlyAdouralcFWFQWF2+NgCHShjkCb+IFBLq9buZwE1xckQU4peSuQ==", + "dev": true, + "requires": { + "ansi-styles": "^3.2.1", + "escape-string-regexp": "^1.0.5", + "supports-color": "^5.3.0" + } + }, + "chardet": { + "version": "0.7.0", + "resolved": "https://registry.npmjs.org/chardet/-/chardet-0.7.0.tgz", + "integrity": "sha512-mT8iDcrh03qDGRRmoA2hmBJnxpllMR+0/0qlzjqZES6NdiWDcZkCNAk4rPFZ9Q85r27unkiNNg8ZOiwZXBHwcA==", + "dev": true + }, + "cli-cursor": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/cli-cursor/-/cli-cursor-3.1.0.tgz", + "integrity": "sha512-I/zHAwsKf9FqGoXM4WWRACob9+SNukZTd94DWF57E4toouRulbCxcUh6RKUEOQlYTHJnzkPMySvPNaaSLNfLZw==", + "dev": true, + "requires": { + "restore-cursor": "^3.1.0" + } + }, + "cli-width": { + "version": "2.2.1", + "resolved": "https://registry.npmjs.org/cli-width/-/cli-width-2.2.1.tgz", + "integrity": "sha512-GRMWDxpOB6Dgk2E5Uo+3eEBvtOOlimMmpbFiKuLFnQzYDavtLFY3K5ona41jgN/WdRZtG7utuVSVTL4HbZHGkw==", + "dev": true + }, + "cliui": { + "version": "5.0.0", + "resolved": "https://registry.npmjs.org/cliui/-/cliui-5.0.0.tgz", + "integrity": "sha512-PYeGSEmmHM6zvoef2w8TPzlrnNpXIjTipYK780YswmIP9vjxmd6Y2a3CB2Ks6/AU8NHjZugXvo8w3oWM2qnwXA==", + "dev": true, + "requires": { + "string-width": "^3.1.0", + "strip-ansi": "^5.2.0", + "wrap-ansi": "^5.1.0" + }, + "dependencies": { + "emoji-regex": { + "version": "7.0.3", + "resolved": "https://registry.npmjs.org/emoji-regex/-/emoji-regex-7.0.3.tgz", + "integrity": "sha512-CwBLREIQ7LvYFB0WyRvwhq5N5qPhc6PMjD6bYggFlI5YyDgl+0vxq5VHbMOFqLg7hfWzmu8T5Z1QofhmTIhItA==", + "dev": true + }, + "is-fullwidth-code-point": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/is-fullwidth-code-point/-/is-fullwidth-code-point-2.0.0.tgz", + "integrity": "sha512-VHskAKYM8RfSFXwee5t5cbN5PZeq1Wrh6qd5bkyiXIf6UQcN6w/A0eXM9r6t8d+GYOh+o6ZhiEnb88LN/Y8m2w==", + "dev": true + }, + "string-width": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/string-width/-/string-width-3.1.0.tgz", + "integrity": "sha512-vafcv6KjVZKSgz06oM/H6GDBrAtz8vdhQakGjFIvNrHA6y3HCF1CInLy+QLq8dTJPQ1b+KDUqDFctkdRW44e1w==", + "dev": true, + "requires": { + "emoji-regex": "^7.0.1", + "is-fullwidth-code-point": "^2.0.0", + "strip-ansi": "^5.1.0" + } + } + } + }, "cluster-key-slot": { "version": "1.1.0", "resolved": "https://registry.npmjs.org/cluster-key-slot/-/cluster-key-slot-1.1.0.tgz", @@ -566,6 +840,21 @@ "resolved": "https://registry.npmjs.org/coffee-script/-/coffee-script-1.6.0.tgz", "integrity": "sha512-Tx8itEfCsQp8RbLDFt7qwjqXycAx2g6SI7//4PPUR2j6meLmNifYm6zKrNDcU1+Q/GWRhjhEZk7DaLG1TfIzGA==" }, + "color-convert": { + "version": "1.9.3", + "resolved": "https://registry.npmjs.org/color-convert/-/color-convert-1.9.3.tgz", + "integrity": "sha512-QfAUtd+vFdAtFQcC8CCyYt1fYWxSqAiK2cSD6zDB8N3cpsEBAvRxp9zOGg6G/SHHJYAT88/az/IuDGALsNVbGg==", + "dev": true, + "requires": { + "color-name": "1.1.3" + } + }, + "color-name": { + "version": "1.1.3", + "resolved": "https://registry.npmjs.org/color-name/-/color-name-1.1.3.tgz", + "integrity": "sha512-72fSenhMw2HZMTVHeCA9KCmpEIbzWiQsjN+BHcBbS9vr1mtt+vJjPdksIBNUmKAW8TFUDPJK5SUU3QhE9NEXDw==", + "dev": true + }, "combined-stream": { "version": "1.0.7", "resolved": "https://registry.npmjs.org/combined-stream/-/combined-stream-1.0.7.tgz", @@ -574,6 +863,12 @@ "delayed-stream": "~1.0.0" } }, + "common-tags": { + "version": "1.8.0", + "resolved": "https://registry.npmjs.org/common-tags/-/common-tags-1.8.0.tgz", + "integrity": "sha512-6P6g0uetGpW/sdyUy/iQQCbFF0kWVMSIVSyYz7Zgjcgh8mgw8PQzDNZeyZ5DQ2gM7LBoZPHmnjz8rUthkBG5tw==", + "dev": true + }, "concat-map": { "version": "0.0.1", "resolved": "https://registry.npmjs.org/concat-map/-/concat-map-0.0.1.tgz", @@ -603,6 +898,12 @@ "resolved": "https://registry.npmjs.org/console-log-level/-/console-log-level-1.4.1.tgz", "integrity": "sha512-VZzbIORbP+PPcN/gg3DXClTLPLg5Slwd5fL2MIc+o1qZ4BXBvWyc6QxPk6T/Mkr6IVjRpoAGf32XxP3ZWMVRcQ==" }, + "contains-path": { + "version": "0.1.0", + "resolved": "https://registry.npmjs.org/contains-path/-/contains-path-0.1.0.tgz", + "integrity": "sha512-OKZnPGeMQy2RPaUIBPFFd71iNf4791H12MCRuVQDnzGRwCYNYmTDy5pdafo2SLAcEMKzTOQnLWG4QdcjeJUMEg==", + "dev": true + }, "content-disposition": { "version": "0.5.3", "resolved": "https://registry.npmjs.org/content-disposition/-/content-disposition-0.5.3.tgz", @@ -664,11 +965,38 @@ "integrity": "sha512-Alvs19Vgq07eunykd3Xy2jF0/qSNv2u7KDbAek9H5liV1UMijbqFs5cycZvv5dVsvseT/U4H8/7/w8Koh35C4A==", "dev": true }, + "core-js": { + "version": "3.6.5", + "resolved": "https://registry.npmjs.org/core-js/-/core-js-3.6.5.tgz", + "integrity": "sha512-vZVEEwZoIsI+vPEuoF9Iqf5H7/M3eeQqWlQnYa8FSKKePuYTf5MWnxb5SDAzCa60b3JBRS5g9b+Dq7b1y/RCrA==", + "dev": true + }, "core-util-is": { "version": "1.0.2", "resolved": "https://registry.npmjs.org/core-util-is/-/core-util-is-1.0.2.tgz", "integrity": "sha512-3lqz5YjWTYnW6dlDa5TLaTCcShfar1e40rmcJVwCBJC6mWlFuj0eCHIElmG1g5kyuJ/GD+8Wn4FFCcz4gJPfaQ==" }, + "cross-spawn": { + "version": "6.0.5", + "resolved": "https://registry.npmjs.org/cross-spawn/-/cross-spawn-6.0.5.tgz", + "integrity": "sha512-eTVLrBSt7fjbDygz805pMnstIs2VTBNkRm0qxZd+M7A5XDdxVRWO5MxGBXZhjY4cqLYLdtrGqRf8mBPmzwSpWQ==", + "dev": true, + "requires": { + "nice-try": "^1.0.4", + "path-key": "^2.0.1", + "semver": "^5.5.0", + "shebang-command": "^1.2.0", + "which": "^1.2.9" + }, + "dependencies": { + "semver": { + "version": "5.7.1", + "resolved": "https://registry.npmjs.org/semver/-/semver-5.7.1.tgz", + "integrity": "sha512-sauaDf/PZdVgrLTNYHRtpXa1iRiKcaebiKQ1BJdpQlWH2lCvexQdX55snPFyK7QzpudqbCI0qXFfOasHdyNDGQ==", + "dev": true + } + } + }, "dashdash": { "version": "1.14.1", "resolved": "https://registry.npmjs.org/dashdash/-/dashdash-1.14.1.tgz", @@ -685,6 +1013,12 @@ "ms": "2.0.0" } }, + "decamelize": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/decamelize/-/decamelize-1.2.0.tgz", + "integrity": "sha512-z2S+W9X73hAUUki+N+9Za2lBlun89zigOyGrsax+KUQ6wKW4ZoWpEYBkGhQjwAjjDCkWxhY0VKEhk8wzY7F5cA==", + "dev": true + }, "deep-eql": { "version": "0.1.3", "resolved": "https://registry.npmjs.org/deep-eql/-/deep-eql-0.1.3.tgz", @@ -694,6 +1028,21 @@ "type-detect": "0.1.1" } }, + "deep-is": { + "version": "0.1.3", + "resolved": "https://registry.npmjs.org/deep-is/-/deep-is-0.1.3.tgz", + "integrity": "sha512-GtxAN4HvBachZzm4OnWqc45ESpUCMwkYcsjnsPs23FwJbsO+k4t0k9bQCgOmzIlpHO28+WPK/KRbRk0DDHuuDw==", + "dev": true + }, + "define-properties": { + "version": "1.1.3", + "resolved": "https://registry.npmjs.org/define-properties/-/define-properties-1.1.3.tgz", + "integrity": "sha512-3MqfYKj2lLzdMSf8ZIZE/V+Zuy+BgD6f164e8K2w7dgnpKArBDerGYpM46IYYcjnkdPNMjPk9A6VFB8+3SKlXQ==", + "dev": true, + "requires": { + "object-keys": "^1.0.12" + } + }, "delay": { "version": "4.3.0", "resolved": "https://registry.npmjs.org/delay/-/delay-4.3.0.tgz", @@ -725,6 +1074,21 @@ "integrity": "sha512-A46qtFgd+g7pDZinpnwiRJtxbC1hpgf0uzP3iG89scHk0AUC7A1TGxf5OiiOUv/JMZR8GOt8hL900hV0bOy5xA==", "dev": true }, + "dlv": { + "version": "1.1.3", + "resolved": "https://registry.npmjs.org/dlv/-/dlv-1.1.3.tgz", + "integrity": "sha512-+HlytyjlPKnIG8XuRG8WvmBP8xs8P71y+SKKS6ZXWoEgLuePxtDoUEiH7WkdePWrQ5JBpE6aoVqfZfJUQkjXwA==", + "dev": true + }, + "doctrine": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/doctrine/-/doctrine-3.0.0.tgz", + "integrity": "sha512-yS+Q5i3hBf7GBkd4KG8a7eBNNWNGLTaEwwYWUijIYM7zrlYDM0BFXHjjPWlWZ1Rg7UaddZeIDmi9jF3HmqiQ2w==", + "dev": true, + "requires": { + "esutils": "^2.0.2" + } + }, "dtrace-provider": { "version": "0.2.8", "resolved": "https://registry.npmjs.org/dtrace-provider/-/dtrace-provider-0.2.8.tgz", @@ -773,6 +1137,12 @@ "shimmer": "^1.2.0" } }, + "emoji-regex": { + "version": "8.0.0", + "resolved": "https://registry.npmjs.org/emoji-regex/-/emoji-regex-8.0.0.tgz", + "integrity": "sha512-MSjYzcWNOA0ewAHpz0MxpYFvwg6yjy1NG3xteoqz644VCo/RPgnr1/GGt+ic3iJTzQ8Eu3TdM14SawnVUmGE6A==", + "dev": true + }, "encodeurl": { "version": "1.0.2", "resolved": "https://registry.npmjs.org/encodeurl/-/encodeurl-1.0.2.tgz", @@ -791,6 +1161,45 @@ "resolved": "https://registry.npmjs.org/ent/-/ent-2.2.0.tgz", "integrity": "sha512-GHrMyVZQWvTIdDtpiEXdHZnFQKzeO09apj8Cbl4pKWy4i0Oprcq17usfDt5aO63swf0JOeMWjWQE/LzgSRuWpA==" }, + "error-ex": { + "version": "1.3.2", + "resolved": "https://registry.npmjs.org/error-ex/-/error-ex-1.3.2.tgz", + "integrity": "sha512-7dFHNmqeFSEt2ZBsCriorKnn3Z2pj+fd9kmI6QoWw4//DL+icEBfc0U7qJCisqrTsKTjw4fNFy2pW9OqStD84g==", + "dev": true, + "requires": { + "is-arrayish": "^0.2.1" + } + }, + "es-abstract": { + "version": "1.17.6", + "resolved": "https://registry.npmjs.org/es-abstract/-/es-abstract-1.17.6.tgz", + "integrity": "sha512-Fr89bON3WFyUi5EvAeI48QTWX0AyekGgLA8H+c+7fbfCkJwRWRMLd8CQedNEyJuoYYhmtEqY92pgte1FAhBlhw==", + "dev": true, + "requires": { + "es-to-primitive": "^1.2.1", + "function-bind": "^1.1.1", + "has": "^1.0.3", + "has-symbols": "^1.0.1", + "is-callable": "^1.2.0", + "is-regex": "^1.1.0", + "object-inspect": "^1.7.0", + "object-keys": "^1.1.1", + "object.assign": "^4.1.0", + "string.prototype.trimend": "^1.0.1", + "string.prototype.trimstart": "^1.0.1" + } + }, + "es-to-primitive": { + "version": "1.2.1", + "resolved": "https://registry.npmjs.org/es-to-primitive/-/es-to-primitive-1.2.1.tgz", + "integrity": "sha512-QCOllgZJtaUo9miYBcLChTUaHNjJF3PYs1VidD7AwiEj1kYxKeQTctLAezAOH5ZKRH0g2IgPn6KwB4IT8iRpvA==", + "dev": true, + "requires": { + "is-callable": "^1.1.4", + "is-date-object": "^1.0.1", + "is-symbol": "^1.0.2" + } + }, "es6-promise": { "version": "4.2.8", "resolved": "https://registry.npmjs.org/es6-promise/-/es6-promise-4.2.8.tgz", @@ -815,6 +1224,345 @@ "integrity": "sha512-vbRorB5FUQWvla16U8R/qgaFIya2qGzwDrNmCZuYKrbdSUMG6I1ZCGQRefkRVhuOkIGVne7BQ35DSfo1qvJqFg==", "dev": true }, + "eslint": { + "version": "6.8.0", + "resolved": "https://registry.npmjs.org/eslint/-/eslint-6.8.0.tgz", + "integrity": "sha512-K+Iayyo2LtyYhDSYwz5D5QdWw0hCacNzyq1Y821Xna2xSJj7cijoLLYmLxTQgcgZ9mC61nryMy9S7GRbYpI5Ig==", + "dev": true, + "requires": { + "@babel/code-frame": "^7.0.0", + "ajv": "^6.10.0", + "chalk": "^2.1.0", + "cross-spawn": "^6.0.5", + "debug": "^4.0.1", + "doctrine": "^3.0.0", + "eslint-scope": "^5.0.0", + "eslint-utils": "^1.4.3", + "eslint-visitor-keys": "^1.1.0", + "espree": "^6.1.2", + "esquery": "^1.0.1", + "esutils": "^2.0.2", + "file-entry-cache": "^5.0.1", + "functional-red-black-tree": "^1.0.1", + "glob-parent": "^5.0.0", + "globals": "^12.1.0", + "ignore": "^4.0.6", + "import-fresh": "^3.0.0", + "imurmurhash": "^0.1.4", + "inquirer": "^7.0.0", + "is-glob": "^4.0.0", + "js-yaml": "^3.13.1", + "json-stable-stringify-without-jsonify": "^1.0.1", + "levn": "^0.3.0", + "lodash": "^4.17.14", + "minimatch": "^3.0.4", + "mkdirp": "^0.5.1", + "natural-compare": "^1.4.0", + "optionator": "^0.8.3", + "progress": "^2.0.0", + "regexpp": "^2.0.1", + "semver": "^6.1.2", + "strip-ansi": "^5.2.0", + "strip-json-comments": "^3.0.1", + "table": "^5.2.3", + "text-table": "^0.2.0", + "v8-compile-cache": "^2.0.3" + }, + "dependencies": { + "debug": { + "version": "4.1.1", + "resolved": "https://registry.npmjs.org/debug/-/debug-4.1.1.tgz", + "integrity": "sha512-pYAIzeRo8J6KPEaJ0VWOh5Pzkbw/RetuzehGM7QRRX5he4fPHx2rdKMB256ehJCkX+XRQm16eZLqLNS8RSZXZw==", + "dev": true, + "requires": { + "ms": "^2.1.1" + } + }, + "ms": { + "version": "2.1.2", + "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.2.tgz", + "integrity": "sha512-sGkPx+VjMtmA6MX27oA4FBFELFCZZ4S4XqeGOXCv68tT+jb3vk/RyaKWP0PTKyWtmLSM0b+adUTEvbs1PEaH2w==", + "dev": true + } + } + }, + "eslint-config-prettier": { + "version": "6.11.0", + "resolved": "https://registry.npmjs.org/eslint-config-prettier/-/eslint-config-prettier-6.11.0.tgz", + "integrity": "sha512-oB8cpLWSAjOVFEJhhyMZh6NOEOtBVziaqdDQ86+qhDHFbZXoRTM7pNSvFRfW/W/L/LrQ38C99J5CGuRBBzBsdA==", + "dev": true, + "requires": { + "get-stdin": "^6.0.0" + } + }, + "eslint-config-standard": { + "version": "14.1.1", + "resolved": "https://registry.npmjs.org/eslint-config-standard/-/eslint-config-standard-14.1.1.tgz", + "integrity": "sha512-Z9B+VR+JIXRxz21udPTL9HpFMyoMUEeX1G251EQ6e05WD9aPVtVBn09XUmZ259wCMlCDmYDSZG62Hhm+ZTJcUg==", + "dev": true + }, + "eslint-import-resolver-node": { + "version": "0.3.4", + "resolved": "https://registry.npmjs.org/eslint-import-resolver-node/-/eslint-import-resolver-node-0.3.4.tgz", + "integrity": "sha512-ogtf+5AB/O+nM6DIeBUNr2fuT7ot9Qg/1harBfBtaP13ekEWFQEEMP94BCB7zaNW3gyY+8SHYF00rnqYwXKWOA==", + "dev": true, + "requires": { + "debug": "^2.6.9", + "resolve": "^1.13.1" + } + }, + "eslint-module-utils": { + "version": "2.6.0", + "resolved": "https://registry.npmjs.org/eslint-module-utils/-/eslint-module-utils-2.6.0.tgz", + "integrity": "sha512-6j9xxegbqe8/kZY8cYpcp0xhbK0EgJlg3g9mib3/miLaExuuwc3n5UEfSnU6hWMbT0FAYVvDbL9RrRgpUeQIvA==", + "dev": true, + "requires": { + "debug": "^2.6.9", + "pkg-dir": "^2.0.0" + } + }, + "eslint-plugin-chai-expect": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/eslint-plugin-chai-expect/-/eslint-plugin-chai-expect-2.1.0.tgz", + "integrity": "sha512-rd0/4mjMV6c3i0o4DKkWI4uaFN9DK707kW+/fDphaDI6HVgxXnhML9Xgt5vHnTXmSSnDhupuCFBgsEAEpchXmQ==", + "dev": true + }, + "eslint-plugin-chai-friendly": { + "version": "0.5.0", + "resolved": "https://registry.npmjs.org/eslint-plugin-chai-friendly/-/eslint-plugin-chai-friendly-0.5.0.tgz", + "integrity": "sha512-Pxe6z8C9fP0pn2X2nGFU/b3GBOCM/5FVus1hsMwJsXP3R7RiXFl7g0ksJbsc0GxiLyidTW4mEFk77qsNn7Tk7g==", + "dev": true + }, + "eslint-plugin-es": { + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/eslint-plugin-es/-/eslint-plugin-es-3.0.1.tgz", + "integrity": "sha512-GUmAsJaN4Fc7Gbtl8uOBlayo2DqhwWvEzykMHSCZHU3XdJ+NSzzZcVhXh3VxX5icqQ+oQdIEawXX8xkR3mIFmQ==", + "dev": true, + "requires": { + "eslint-utils": "^2.0.0", + "regexpp": "^3.0.0" + }, + "dependencies": { + "eslint-utils": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/eslint-utils/-/eslint-utils-2.1.0.tgz", + "integrity": "sha512-w94dQYoauyvlDc43XnGB8lU3Zt713vNChgt4EWwhXAP2XkBvndfxF0AgIqKOOasjPIPzj9JqgwkwbCYD0/V3Zg==", + "dev": true, + "requires": { + "eslint-visitor-keys": "^1.1.0" + } + }, + "regexpp": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/regexpp/-/regexpp-3.1.0.tgz", + "integrity": "sha512-ZOIzd8yVsQQA7j8GCSlPGXwg5PfmA1mrq0JP4nGhh54LaKN3xdai/vHUDu74pKwV8OxseMS65u2NImosQcSD0Q==", + "dev": true + } + } + }, + "eslint-plugin-import": { + "version": "2.21.2", + "resolved": "https://registry.npmjs.org/eslint-plugin-import/-/eslint-plugin-import-2.21.2.tgz", + "integrity": "sha512-FEmxeGI6yaz+SnEB6YgNHlQK1Bs2DKLM+YF+vuTk5H8J9CLbJLtlPvRFgZZ2+sXiKAlN5dpdlrWOjK8ZoZJpQA==", + "dev": true, + "requires": { + "array-includes": "^3.1.1", + "array.prototype.flat": "^1.2.3", + "contains-path": "^0.1.0", + "debug": "^2.6.9", + "doctrine": "1.5.0", + "eslint-import-resolver-node": "^0.3.3", + "eslint-module-utils": "^2.6.0", + "has": "^1.0.3", + "minimatch": "^3.0.4", + "object.values": "^1.1.1", + "read-pkg-up": "^2.0.0", + "resolve": "^1.17.0", + "tsconfig-paths": "^3.9.0" + }, + "dependencies": { + "doctrine": { + "version": "1.5.0", + "resolved": "https://registry.npmjs.org/doctrine/-/doctrine-1.5.0.tgz", + "integrity": "sha512-lsGyRuYr4/PIB0txi+Fy2xOMI2dGaTguCaotzFGkVZuKR5usKfcRWIFKNM3QNrU7hh/+w2bwTW+ZeXPK5l8uVg==", + "dev": true, + "requires": { + "esutils": "^2.0.2", + "isarray": "^1.0.0" + } + }, + "resolve": { + "version": "1.17.0", + "resolved": "https://registry.npmjs.org/resolve/-/resolve-1.17.0.tgz", + "integrity": "sha512-ic+7JYiV8Vi2yzQGFWOkiZD5Z9z7O2Zhm9XMaTxdJExKasieFCr+yXZ/WmXsckHiKl12ar0y6XiXDx3m4RHn1w==", + "dev": true, + "requires": { + "path-parse": "^1.0.6" + } + } + } + }, + "eslint-plugin-mocha": { + "version": "6.3.0", + "resolved": "https://registry.npmjs.org/eslint-plugin-mocha/-/eslint-plugin-mocha-6.3.0.tgz", + "integrity": "sha512-Cd2roo8caAyG21oKaaNTj7cqeYRWW1I2B5SfpKRp0Ip1gkfwoR1Ow0IGlPWnNjzywdF4n+kHL8/9vM6zCJUxdg==", + "dev": true, + "requires": { + "eslint-utils": "^2.0.0", + "ramda": "^0.27.0" + }, + "dependencies": { + "eslint-utils": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/eslint-utils/-/eslint-utils-2.1.0.tgz", + "integrity": "sha512-w94dQYoauyvlDc43XnGB8lU3Zt713vNChgt4EWwhXAP2XkBvndfxF0AgIqKOOasjPIPzj9JqgwkwbCYD0/V3Zg==", + "dev": true, + "requires": { + "eslint-visitor-keys": "^1.1.0" + } + } + } + }, + "eslint-plugin-node": { + "version": "11.1.0", + "resolved": "https://registry.npmjs.org/eslint-plugin-node/-/eslint-plugin-node-11.1.0.tgz", + "integrity": "sha512-oUwtPJ1W0SKD0Tr+wqu92c5xuCeQqB3hSCHasn/ZgjFdA9iDGNkNf2Zi9ztY7X+hNuMib23LNGRm6+uN+KLE3g==", + "dev": true, + "requires": { + "eslint-plugin-es": "^3.0.0", + "eslint-utils": "^2.0.0", + "ignore": "^5.1.1", + "minimatch": "^3.0.4", + "resolve": "^1.10.1", + "semver": "^6.1.0" + }, + "dependencies": { + "eslint-utils": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/eslint-utils/-/eslint-utils-2.1.0.tgz", + "integrity": "sha512-w94dQYoauyvlDc43XnGB8lU3Zt713vNChgt4EWwhXAP2XkBvndfxF0AgIqKOOasjPIPzj9JqgwkwbCYD0/V3Zg==", + "dev": true, + "requires": { + "eslint-visitor-keys": "^1.1.0" + } + }, + "ignore": { + "version": "5.1.8", + "resolved": "https://registry.npmjs.org/ignore/-/ignore-5.1.8.tgz", + "integrity": "sha512-BMpfD7PpiETpBl/A6S498BaIJ6Y/ABT93ETbby2fP00v4EbvPBXWEoaR1UBPKs3iR53pJY7EtZk5KACI57i1Uw==", + "dev": true + } + } + }, + "eslint-plugin-prettier": { + "version": "3.1.4", + "resolved": "https://registry.npmjs.org/eslint-plugin-prettier/-/eslint-plugin-prettier-3.1.4.tgz", + "integrity": "sha512-jZDa8z76klRqo+TdGDTFJSavwbnWK2ZpqGKNZ+VvweMW516pDUMmQ2koXvxEE4JhzNvTv+radye/bWGBmA6jmg==", + "dev": true, + "requires": { + "prettier-linter-helpers": "^1.0.0" + } + }, + "eslint-plugin-promise": { + "version": "4.2.1", + "resolved": "https://registry.npmjs.org/eslint-plugin-promise/-/eslint-plugin-promise-4.2.1.tgz", + "integrity": "sha512-VoM09vT7bfA7D+upt+FjeBO5eHIJQBUWki1aPvB+vbNiHS3+oGIJGIeyBtKQTME6UPXXy3vV07OL1tHd3ANuDw==", + "dev": true + }, + "eslint-plugin-standard": { + "version": "4.0.1", + "resolved": "https://registry.npmjs.org/eslint-plugin-standard/-/eslint-plugin-standard-4.0.1.tgz", + "integrity": "sha512-v/KBnfyaOMPmZc/dmc6ozOdWqekGp7bBGq4jLAecEfPGmfKiWS4sA8sC0LqiV9w5qmXAtXVn4M3p1jSyhY85SQ==", + "dev": true + }, + "eslint-scope": { + "version": "5.1.0", + "resolved": "https://registry.npmjs.org/eslint-scope/-/eslint-scope-5.1.0.tgz", + "integrity": "sha512-iiGRvtxWqgtx5m8EyQUJihBloE4EnYeGE/bz1wSPwJE6tZuJUtHlhqDM4Xj2ukE8Dyy1+HCZ4hE0fzIVMzb58w==", + "dev": true, + "requires": { + "esrecurse": "^4.1.0", + "estraverse": "^4.1.1" + } + }, + "eslint-utils": { + "version": "1.4.3", + "resolved": "https://registry.npmjs.org/eslint-utils/-/eslint-utils-1.4.3.tgz", + "integrity": "sha512-fbBN5W2xdY45KulGXmLHZ3c3FHfVYmKg0IrAKGOkT/464PQsx2UeIzfz1RmEci+KLm1bBaAzZAh8+/E+XAeZ8Q==", + "dev": true, + "requires": { + "eslint-visitor-keys": "^1.1.0" + } + }, + "eslint-visitor-keys": { + "version": "1.3.0", + "resolved": "https://registry.npmjs.org/eslint-visitor-keys/-/eslint-visitor-keys-1.3.0.tgz", + "integrity": "sha512-6J72N8UNa462wa/KFODt/PJ3IU60SDpC3QXC1Hjc1BXXpfL2C9R5+AU7jhe0F6GREqVMh4Juu+NY7xn+6dipUQ==", + "dev": true + }, + "espree": { + "version": "6.2.1", + "resolved": "https://registry.npmjs.org/espree/-/espree-6.2.1.tgz", + "integrity": "sha512-ysCxRQY3WaXJz9tdbWOwuWr5Y/XrPTGX9Kiz3yoUXwW0VZ4w30HTkQLaGx/+ttFjF8i+ACbArnB4ce68a9m5hw==", + "dev": true, + "requires": { + "acorn": "^7.1.1", + "acorn-jsx": "^5.2.0", + "eslint-visitor-keys": "^1.1.0" + }, + "dependencies": { + "acorn": { + "version": "7.3.1", + "resolved": "https://registry.npmjs.org/acorn/-/acorn-7.3.1.tgz", + "integrity": "sha512-tLc0wSnatxAQHVHUapaHdz72pi9KUyHjq5KyHjGg9Y8Ifdc79pTh2XvI6I1/chZbnM7QtNKzh66ooDogPZSleA==", + "dev": true + } + } + }, + "esprima": { + "version": "4.0.1", + "resolved": "https://registry.npmjs.org/esprima/-/esprima-4.0.1.tgz", + "integrity": "sha512-eGuFFw7Upda+g4p+QHvnW0RyTX/SVeJBDM/gCtMARO0cLuT2HcEKnTPvhjV6aGeqrCB/sbNop0Kszm0jsaWU4A==", + "dev": true + }, + "esquery": { + "version": "1.3.1", + "resolved": "https://registry.npmjs.org/esquery/-/esquery-1.3.1.tgz", + "integrity": "sha512-olpvt9QG0vniUBZspVRN6lwB7hOZoTRtT+jzR+tS4ffYx2mzbw+z0XCOk44aaLYKApNX5nMm+E+P6o25ip/DHQ==", + "dev": true, + "requires": { + "estraverse": "^5.1.0" + }, + "dependencies": { + "estraverse": { + "version": "5.1.0", + "resolved": "https://registry.npmjs.org/estraverse/-/estraverse-5.1.0.tgz", + "integrity": "sha512-FyohXK+R0vE+y1nHLoBM7ZTyqRpqAlhdZHCWIWEviFLiGB8b04H6bQs8G+XTthacvT8VuwvteiP7RJSxMs8UEw==", + "dev": true + } + } + }, + "esrecurse": { + "version": "4.2.1", + "resolved": "https://registry.npmjs.org/esrecurse/-/esrecurse-4.2.1.tgz", + "integrity": "sha512-64RBB++fIOAXPw3P9cy89qfMlvZEXZkqqJkjqqXIvzP5ezRZjW+lPWjw35UX/3EhUPFYbg5ER4JYgDw4007/DQ==", + "dev": true, + "requires": { + "estraverse": "^4.1.0" + } + }, + "estraverse": { + "version": "4.3.0", + "resolved": "https://registry.npmjs.org/estraverse/-/estraverse-4.3.0.tgz", + "integrity": "sha512-39nnKffWz8xN1BU/2c79n9nB9HDzo0niYUqx6xyqUnyoAnQyyWpOTdZEeiCch8BBu515t4wp9ZmgVfVhn9EBpw==", + "dev": true + }, + "esutils": { + "version": "2.0.3", + "resolved": "https://registry.npmjs.org/esutils/-/esutils-2.0.3.tgz", + "integrity": "sha512-kVscqXk4OCp68SZ0dkgEKVi6/8ij300KBWTJq32P/dYeWTSwK41WyTxalN1eRmA5Z9UU/LX9D7FWSmV9SAYx6g==", + "dev": true + }, "etag": { "version": "1.8.1", "resolved": "https://registry.npmjs.org/etag/-/etag-1.8.1.tgz", @@ -921,6 +1669,17 @@ "resolved": "https://registry.npmjs.org/extend/-/extend-3.0.2.tgz", "integrity": "sha512-fjquC59cD7CyW6urNXK0FBufkZcoiGG80wTuPujX590cB5Ttln20E2UB4S/WARVqhXffZl2LNgS+gQdPIIim/g==" }, + "external-editor": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/external-editor/-/external-editor-3.1.0.tgz", + "integrity": "sha512-hMQ4CX1p1izmuLYyZqLMO/qGNw10wSv9QDCPfzXfyFrOaCSSoRfqE1Kf1s5an66J5JZC62NewG+mK49jOCtQew==", + "dev": true, + "requires": { + "chardet": "^0.7.0", + "iconv-lite": "^0.4.24", + "tmp": "^0.0.33" + } + }, "extsprintf": { "version": "1.3.0", "resolved": "https://registry.npmjs.org/extsprintf/-/extsprintf-1.3.0.tgz", @@ -931,16 +1690,46 @@ "resolved": "https://registry.npmjs.org/fast-deep-equal/-/fast-deep-equal-3.1.3.tgz", "integrity": "sha512-f3qQ9oQy9j2AhBe/H9VC91wLmKBCCU/gDOnKNAYG5hswO7BLKj09Hc5HYNz9cGI++xlpDCIgDaitVs03ATR84Q==" }, + "fast-diff": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/fast-diff/-/fast-diff-1.2.0.tgz", + "integrity": "sha512-xJuoT5+L99XlZ8twedaRf6Ax2TgQVxvgZOYoPKqZufmJib0tL2tegPBOZb1pVNgIhlqDlA0eO0c3wBvQcmzx4w==", + "dev": true + }, "fast-json-stable-stringify": { "version": "2.1.0", "resolved": "https://registry.npmjs.org/fast-json-stable-stringify/-/fast-json-stable-stringify-2.1.0.tgz", "integrity": "sha512-lhd/wF+Lk98HZoTCtlVraHtfh5XYijIjalXck7saUtuanSDyLMxnHhSXEDJqHxD7msR8D0uCmqlkwjCV8xvwHw==" }, + "fast-levenshtein": { + "version": "2.0.6", + "resolved": "https://registry.npmjs.org/fast-levenshtein/-/fast-levenshtein-2.0.6.tgz", + "integrity": "sha512-DCXu6Ifhqcks7TZKY3Hxp3y6qphY5SJZmrWMDrKcERSOXWQdMhU9Ig/PYrzyw/ul9jOIyh0N4M0tbC5hodg8dw==", + "dev": true + }, "fast-text-encoding": { "version": "1.0.1", "resolved": "https://registry.npmjs.org/fast-text-encoding/-/fast-text-encoding-1.0.1.tgz", "integrity": "sha512-x4FEgaz3zNRtJfLFqJmHWxkMDDvXVtaznj2V9jiP8ACUJrUgist4bP9FmDL2Vew2Y9mEQI/tG4GqabaitYp9CQ==" }, + "figures": { + "version": "3.2.0", + "resolved": "https://registry.npmjs.org/figures/-/figures-3.2.0.tgz", + "integrity": "sha512-yaduQFRKLXYOGgEn6AZau90j3ggSOyiqXU0F9JZfeXYhNa+Jk4X+s45A2zg5jns87GAFa34BBm2kXw4XpNcbdg==", + "dev": true, + "requires": { + "escape-string-regexp": "^1.0.5" + } + }, + "file-entry-cache": { + "version": "5.0.1", + "resolved": "https://registry.npmjs.org/file-entry-cache/-/file-entry-cache-5.0.1.tgz", + "integrity": "sha512-bCg29ictuBaKUwwArK4ouCaqDgLZcysCFLmM/Yn/FDoqndh/9vNuQfXRDvTuXKLxfD/JtZQGKFT8MGcJBK644g==", + "dev": true, + "requires": { + "flat-cache": "^2.0.1" + } + }, "file-uri-to-path": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/file-uri-to-path/-/file-uri-to-path-1.0.0.tgz", @@ -960,11 +1749,62 @@ "unpipe": "~1.0.0" } }, + "find-up": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/find-up/-/find-up-2.1.0.tgz", + "integrity": "sha512-NWzkk0jSJtTt08+FBFMvXoeZnOJD+jTtsRmBYbAIzJdX6l7dLgR7CTubCM5/eDdPUBvLCeVasP1brfVR/9/EZQ==", + "dev": true, + "requires": { + "locate-path": "^2.0.0" + } + }, "findit2": { "version": "2.2.3", "resolved": "https://registry.npmjs.org/findit2/-/findit2-2.2.3.tgz", "integrity": "sha512-lg/Moejf4qXovVutL0Lz4IsaPoNYMuxt4PA0nGqFxnJ1CTTGGlEO2wKgoDpwknhvZ8k4Q2F+eesgkLbG2Mxfog==" }, + "flat-cache": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/flat-cache/-/flat-cache-2.0.1.tgz", + "integrity": "sha512-LoQe6yDuUMDzQAEH8sgmh4Md6oZnc/7PjtwjNFSzveXqSHt6ka9fPBuso7IGf9Rz4uqnSnWiFH2B/zj24a5ReA==", + "dev": true, + "requires": { + "flatted": "^2.0.0", + "rimraf": "2.6.3", + "write": "1.0.3" + }, + "dependencies": { + "glob": { + "version": "7.1.6", + "resolved": "https://registry.npmjs.org/glob/-/glob-7.1.6.tgz", + "integrity": "sha512-LwaxwyZ72Lk7vZINtNNrywX0ZuLyStrdDtabefZKAY5ZGJhVtgdznluResxNmPitE0SAO+O26sWTHeKSI2wMBA==", + "dev": true, + "requires": { + "fs.realpath": "^1.0.0", + "inflight": "^1.0.4", + "inherits": "2", + "minimatch": "^3.0.4", + "once": "^1.3.0", + "path-is-absolute": "^1.0.0" + } + }, + "rimraf": { + "version": "2.6.3", + "resolved": "https://registry.npmjs.org/rimraf/-/rimraf-2.6.3.tgz", + "integrity": "sha512-mwqeW5XsA2qAejG46gYdENaxXjx9onRNCfn7L0duuP4hCuTIi/QO7PDK07KJfp1d+izWPrzEJDcSqBa0OZQriA==", + "dev": true, + "requires": { + "glob": "^7.1.3" + } + } + } + }, + "flatted": { + "version": "2.0.2", + "resolved": "https://registry.npmjs.org/flatted/-/flatted-2.0.2.tgz", + "integrity": "sha512-r5wGx7YeOwNWNlCA0wQ86zKyDLMQr+/RB8xy74M4hTphfmjlijTSSXGuH8rnvKZnfT9i+75zmd8jcKdMR4O6jA==", + "dev": true + }, "follow-redirects": { "version": "1.5.10", "resolved": "https://registry.npmjs.org/follow-redirects/-/follow-redirects-1.5.10.tgz", @@ -1023,6 +1863,18 @@ "integrity": "sha512-OO0pH2lK6a0hZnAdau5ItzHPI6pUlvI7jMVnxUQRtw4owF2wk8lOSabtGDCTP4Ggrg2MbGnWO9X8K1t4+fGMDw==", "dev": true }, + "function-bind": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/function-bind/-/function-bind-1.1.1.tgz", + "integrity": "sha512-yIovAzMX49sF8Yl58fSCWJ5svSLuaibPxXQJFLmBObTuCr0Mf1KiPopGM9NiFjiYBCbfaa2Fh6breQ6ANVTI0A==", + "dev": true + }, + "functional-red-black-tree": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/functional-red-black-tree/-/functional-red-black-tree-1.0.1.tgz", + "integrity": "sha512-dsKNQNdj6xA3T+QlADDA7mOSlX0qiMINjn0cgr+eGHGsbSHzTabcIogz2+p/iqP1Xs6EP/sS2SbqH+brGTbq0g==", + "dev": true + }, "gaxios": { "version": "1.8.4", "resolved": "https://registry.npmjs.org/gaxios/-/gaxios-1.8.4.tgz", @@ -1043,6 +1895,18 @@ "json-bigint": "^0.3.0" } }, + "get-caller-file": { + "version": "2.0.5", + "resolved": "https://registry.npmjs.org/get-caller-file/-/get-caller-file-2.0.5.tgz", + "integrity": "sha512-DyFP3BM/3YHTQOCUL/w0OZHR0lpKeGrxotcHWcqNEdnltqFwXVfhEBQ94eIo34AfQpo0rGki4cyIiftY06h2Fg==", + "dev": true + }, + "get-stdin": { + "version": "6.0.0", + "resolved": "https://registry.npmjs.org/get-stdin/-/get-stdin-6.0.0.tgz", + "integrity": "sha512-jp4tHawyV7+fkkSKyvjuLZswblUtz+SQKzSWnBbii16BuZksJlU1wuBYXY75r+duh/llF1ur6oNwi+2ZzjKZ7g==", + "dev": true + }, "getpass": { "version": "0.1.7", "resolved": "https://registry.npmjs.org/getpass/-/getpass-0.1.7.tgz", @@ -1064,6 +1928,24 @@ "path-is-absolute": "^1.0.0" } }, + "glob-parent": { + "version": "5.1.1", + "resolved": "https://registry.npmjs.org/glob-parent/-/glob-parent-5.1.1.tgz", + "integrity": "sha512-FnI+VGOpnlGHWZxthPGR+QhR78fuiK0sNLkHQv+bL9fQi57lNNdquIbna/WrfROrolq8GK5Ek6BiMwqL/voRYQ==", + "dev": true, + "requires": { + "is-glob": "^4.0.1" + } + }, + "globals": { + "version": "12.4.0", + "resolved": "https://registry.npmjs.org/globals/-/globals-12.4.0.tgz", + "integrity": "sha512-BWICuzzDvDoH54NHKCseDanAhE3CeDorgDL5MT6LMXXj2WCnd9UC2szdk4AWLfjdgNBCXLUanXYcpBBKOSWGwg==", + "dev": true, + "requires": { + "type-fest": "^0.8.1" + } + }, "google-auth-library": { "version": "3.1.2", "resolved": "https://registry.npmjs.org/google-auth-library/-/google-auth-library-3.1.2.tgz", @@ -1096,6 +1978,12 @@ "pify": "^4.0.0" } }, + "graceful-fs": { + "version": "4.2.4", + "resolved": "https://registry.npmjs.org/graceful-fs/-/graceful-fs-4.2.4.tgz", + "integrity": "sha512-WjKPNJF79dtJAVniUlGGWHYGz2jWxT6VhN/4m1NdkbZ2nOsEF+cI1Edgql5zCRhs/VsQYRvrXctxktVXZUkixw==", + "dev": true + }, "gtoken": { "version": "2.3.3", "resolved": "https://registry.npmjs.org/gtoken/-/gtoken-2.3.3.tgz", @@ -1129,6 +2017,44 @@ "har-schema": "^2.0.0" } }, + "has": { + "version": "1.0.3", + "resolved": "https://registry.npmjs.org/has/-/has-1.0.3.tgz", + "integrity": "sha512-f2dvO0VU6Oej7RkWJGrehjbzMAjFp5/VKPp5tTpWIV4JHHZK1/BxbFRtf/siA2SWTe09caDmVtYYzWEIbBS4zw==", + "dev": true, + "requires": { + "function-bind": "^1.1.1" + } + }, + "has-ansi": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/has-ansi/-/has-ansi-2.0.0.tgz", + "integrity": "sha512-C8vBJ8DwUCx19vhm7urhTuUsr4/IyP6l4VzNQDv+ryHQObW3TTTp9yB68WpYgRe2bbaGuZ/se74IqFeVnMnLZg==", + "dev": true, + "requires": { + "ansi-regex": "^2.0.0" + }, + "dependencies": { + "ansi-regex": { + "version": "2.1.1", + "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-2.1.1.tgz", + "integrity": "sha512-TIGnTpdo+E3+pCyAluZvtED5p5wCqLdezCyhPZzKPcxvFplEt4i+W7OONCKgeZFT3+y5NZZfOOS/Bdcanm1MYA==", + "dev": true + } + } + }, + "has-flag": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/has-flag/-/has-flag-3.0.0.tgz", + "integrity": "sha512-sKJf1+ceQBr4SMkvQnBDNDtf4TXpVhVGateu0t918bl30FnbE2m4vNLX+VWe/dpjlb+HugGYzW7uQXH98HPEYw==", + "dev": true + }, + "has-symbols": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/has-symbols/-/has-symbols-1.0.1.tgz", + "integrity": "sha512-PLcsoqu++dmEIZB+6totNFKq/7Do+Z0u4oT0zKOJNl3lYK6vGwwu2hjHs+68OEZbTjiUE9bgOABXbP/GvrS0Kg==", + "dev": true + }, "he": { "version": "1.1.1", "resolved": "https://registry.npmjs.org/he/-/he-1.1.1.tgz", @@ -1140,6 +2066,12 @@ "resolved": "https://registry.npmjs.org/hex2dec/-/hex2dec-1.1.2.tgz", "integrity": "sha512-Yu+q/XWr2fFQ11tHxPq4p4EiNkb2y+lAacJNhAdRXVfRIcDH6gi7htWFnnlIzvqHMHoWeIsfXlNAjZInpAOJDA==" }, + "hosted-git-info": { + "version": "2.8.8", + "resolved": "https://registry.npmjs.org/hosted-git-info/-/hosted-git-info-2.8.8.tgz", + "integrity": "sha512-f/wzC2QaWBs7t9IYqB4T3sR1xviIViXJRJTWBlx2Gf3g0Xi5vI7Yy4koXQ1c9OYDGHN9sBy1DQ2AB8fqZBWhUg==", + "dev": true + }, "http-errors": { "version": "1.7.2", "resolved": "https://registry.npmjs.org/http-errors/-/http-errors-1.7.2.tgz", @@ -1194,6 +2126,34 @@ "safer-buffer": ">= 2.1.2 < 3" } }, + "ignore": { + "version": "4.0.6", + "resolved": "https://registry.npmjs.org/ignore/-/ignore-4.0.6.tgz", + "integrity": "sha512-cyFDKrqc/YdcWFniJhzI42+AzS+gNwmUzOSFcRCQYwySuBBBy/KjuxWLZ/FHEH6Moq1NizMOBWyTcv8O4OZIMg==", + "dev": true + }, + "import-fresh": { + "version": "3.2.1", + "resolved": "https://registry.npmjs.org/import-fresh/-/import-fresh-3.2.1.tgz", + "integrity": "sha512-6e1q1cnWP2RXD9/keSkxHScg508CdXqXWgWBaETNhyuBFz+kUZlKboh+ISK+bU++DmbHimVBrOz/zzPe0sZ3sQ==", + "dev": true, + "requires": { + "parent-module": "^1.0.0", + "resolve-from": "^4.0.0" + } + }, + "imurmurhash": { + "version": "0.1.4", + "resolved": "https://registry.npmjs.org/imurmurhash/-/imurmurhash-0.1.4.tgz", + "integrity": "sha512-JmXMZ6wuvDmLiHEml9ykzqO6lwFbof0GG4IkcGaENdCRDDmMVnny7s5HsIgHCbaq0w2MyPhDqkhTUgS2LU2PHA==", + "dev": true + }, + "indent-string": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/indent-string/-/indent-string-4.0.0.tgz", + "integrity": "sha512-EdDDZu4A2OyIK7Lr/2zG+w5jmbuk1DVBnEwREQvBzspBJkCEbRa8GxU1lghYcaGJCnRWibjDXlq779X1/y5xwg==", + "dev": true + }, "inflight": { "version": "1.0.6", "resolved": "https://registry.npmjs.org/inflight/-/inflight-1.0.6.tgz", @@ -1208,34 +2168,85 @@ "resolved": "https://registry.npmjs.org/inherits/-/inherits-2.0.3.tgz", "integrity": "sha512-x00IRNXNy63jwGkJmzPigoySHbaqpNuzKbBOmzK+g2OdZpQ9w+sxCN+VSB3ja7IAge2OP2qpfxTjeNcyjmW1uw==" }, - "ioredis": { - "version": "4.17.3", - "resolved": "https://registry.npmjs.org/ioredis/-/ioredis-4.17.3.tgz", - "integrity": "sha512-iRvq4BOYzNFkDnSyhx7cmJNOi1x/HWYe+A4VXHBu4qpwJaGT1Mp+D2bVGJntH9K/Z/GeOM/Nprb8gB3bmitz1Q==", + "inquirer": { + "version": "7.2.0", + "resolved": "https://registry.npmjs.org/inquirer/-/inquirer-7.2.0.tgz", + "integrity": "sha512-E0c4rPwr9ByePfNlTIB8z51kK1s2n6jrHuJeEHENl/sbq2G/S1auvibgEwNR4uSyiU+PiYHqSwsgGiXjG8p5ZQ==", + "dev": true, "requires": { - "cluster-key-slot": "^1.1.0", - "debug": "^4.1.1", - "denque": "^1.1.0", - "lodash.defaults": "^4.2.0", - "lodash.flatten": "^4.4.0", - "redis-commands": "1.5.0", - "redis-errors": "^1.2.0", - "redis-parser": "^3.0.0", - "standard-as-callback": "^2.0.1" + "ansi-escapes": "^4.2.1", + "chalk": "^3.0.0", + "cli-cursor": "^3.1.0", + "cli-width": "^2.0.0", + "external-editor": "^3.0.3", + "figures": "^3.0.0", + "lodash": "^4.17.15", + "mute-stream": "0.0.8", + "run-async": "^2.4.0", + "rxjs": "^6.5.3", + "string-width": "^4.1.0", + "strip-ansi": "^6.0.0", + "through": "^2.3.6" }, "dependencies": { - "debug": { - "version": "4.1.1", - "resolved": "https://registry.npmjs.org/debug/-/debug-4.1.1.tgz", - "integrity": "sha512-pYAIzeRo8J6KPEaJ0VWOh5Pzkbw/RetuzehGM7QRRX5he4fPHx2rdKMB256ehJCkX+XRQm16eZLqLNS8RSZXZw==", + "ansi-styles": { + "version": "4.2.1", + "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-4.2.1.tgz", + "integrity": "sha512-9VGjrMsG1vePxcSweQsN20KY/c4zN0h9fLjqAbwbPfahM3t+NL+M9HC8xeXG2I8pX5NoamTGNuomEUFI7fcUjA==", + "dev": true, "requires": { - "ms": "^2.1.1" + "@types/color-name": "^1.1.1", + "color-convert": "^2.0.1" } }, - "ms": { - "version": "2.1.2", - "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.2.tgz", - "integrity": "sha512-sGkPx+VjMtmA6MX27oA4FBFELFCZZ4S4XqeGOXCv68tT+jb3vk/RyaKWP0PTKyWtmLSM0b+adUTEvbs1PEaH2w==" + "chalk": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/chalk/-/chalk-3.0.0.tgz", + "integrity": "sha512-4D3B6Wf41KOYRFdszmDqMCGq5VV/uMAB273JILmO+3jAlh8X4qDtdtgCR3fxtbLEMzSx22QdhnDcJvu2u1fVwg==", + "dev": true, + "requires": { + "ansi-styles": "^4.1.0", + "supports-color": "^7.1.0" + } + }, + "color-convert": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/color-convert/-/color-convert-2.0.1.tgz", + "integrity": "sha512-RRECPsj7iu/xb5oKYcsFHSppFNnsj/52OVTRKb4zP5onXwVF3zVmmToNcOfGC+CRDpfK/U584fMg38ZHCaElKQ==", + "dev": true, + "requires": { + "color-name": "~1.1.4" + } + }, + "color-name": { + "version": "1.1.4", + "resolved": "https://registry.npmjs.org/color-name/-/color-name-1.1.4.tgz", + "integrity": "sha512-dOy+3AuW3a2wNbZHIuMZpTcgjGuLU/uBL/ubcZF9OXbDo8ff4O8yVp5Bf0efS8uEoYo5q4Fx7dY9OgQGXgAsQA==", + "dev": true + }, + "has-flag": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/has-flag/-/has-flag-4.0.0.tgz", + "integrity": "sha512-EykJT/Q1KjTWctppgIAgfSO0tKVuZUjhgMr17kqTumMl6Afv3EISleU7qZUzoXDFTAHTDC4NOoG/ZxU3EvlMPQ==", + "dev": true + }, + "strip-ansi": { + "version": "6.0.0", + "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-6.0.0.tgz", + "integrity": "sha512-AuvKTrTfQNYNIctbR1K/YGTR1756GycPsg7b9bdV9Duqur4gv6aKqHXah67Z8ImS7WEz5QVcOtlfW2rZEugt6w==", + "dev": true, + "requires": { + "ansi-regex": "^5.0.0" + } + }, + "supports-color": { + "version": "7.1.0", + "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-7.1.0.tgz", + "integrity": "sha512-oRSIpR8pxT1Wr2FquTNnGet79b3BWljqOuoW/h4oBhxJ/HUbX5nX6JSruTkvXDCFMwDPvsaTTbvMLKZWSy0R5g==", + "dev": true, + "requires": { + "has-flag": "^4.0.0" + } } } }, @@ -1249,11 +2260,74 @@ "resolved": "https://registry.npmjs.org/is/-/is-3.3.0.tgz", "integrity": "sha512-nW24QBoPcFGGHJGUwnfpI7Yc5CdqWNdsyHQszVE/z2pKHXzh7FZ5GWhJqSyaQ9wMkQnsTx+kAI8bHlCX4tKdbg==" }, + "is-arrayish": { + "version": "0.2.1", + "resolved": "https://registry.npmjs.org/is-arrayish/-/is-arrayish-0.2.1.tgz", + "integrity": "sha512-zz06S8t0ozoDXMG+ube26zeCTNXcKIPJZJi8hBrF4idCLms4CG9QtK7qBl1boi5ODzFpjswb5JPmHCbMpjaYzg==", + "dev": true + }, "is-buffer": { "version": "2.0.4", "resolved": "https://registry.npmjs.org/is-buffer/-/is-buffer-2.0.4.tgz", "integrity": "sha512-Kq1rokWXOPXWuaMAqZiJW4XxsmD9zGx9q4aePabbn3qCRGedtH7Cm+zV8WETitMfu1wdh+Rvd6w5egwSngUX2A==" }, + "is-callable": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/is-callable/-/is-callable-1.2.0.tgz", + "integrity": "sha512-pyVD9AaGLxtg6srb2Ng6ynWJqkHU9bEM087AKck0w8QwDarTfNcpIYoU8x8Hv2Icm8u6kFJM18Dag8lyqGkviw==", + "dev": true + }, + "is-date-object": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/is-date-object/-/is-date-object-1.0.2.tgz", + "integrity": "sha512-USlDT524woQ08aoZFzh3/Z6ch9Y/EWXEHQ/AaRN0SkKq4t2Jw2R2339tSXmwuVoY7LLlBCbOIlx2myP/L5zk0g==", + "dev": true + }, + "is-extglob": { + "version": "2.1.1", + "resolved": "https://registry.npmjs.org/is-extglob/-/is-extglob-2.1.1.tgz", + "integrity": "sha512-SbKbANkN603Vi4jEZv49LeVJMn4yGwsbzZworEoyEiutsN3nJYdbO36zfhGJ6QEDpOZIFkDtnq5JRxmvl3jsoQ==", + "dev": true + }, + "is-fullwidth-code-point": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/is-fullwidth-code-point/-/is-fullwidth-code-point-3.0.0.tgz", + "integrity": "sha512-zymm5+u+sCsSWyD9qNaejV3DFvhCKclKdizYaJUuHA83RLjb7nSuGnddCHGv0hk+KY7BMAlsWeK4Ueg6EV6XQg==", + "dev": true + }, + "is-glob": { + "version": "4.0.1", + "resolved": "https://registry.npmjs.org/is-glob/-/is-glob-4.0.1.tgz", + "integrity": "sha512-5G0tKtBTFImOqDnLB2hG6Bp2qcKEFduo4tZu9MT/H6NQv/ghhy30o55ufafxJ/LdH79LLs2Kfrn85TLKyA7BUg==", + "dev": true, + "requires": { + "is-extglob": "^2.1.1" + } + }, + "is-regex": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/is-regex/-/is-regex-1.1.0.tgz", + "integrity": "sha512-iI97M8KTWID2la5uYXlkbSDQIg4F6o1sYboZKKTDpnDQMLtUL86zxhgDet3Q2SriaYsyGqZ6Mn2SjbRKeLHdqw==", + "dev": true, + "requires": { + "has-symbols": "^1.0.1" + } + }, + "is-string": { + "version": "1.0.5", + "resolved": "https://registry.npmjs.org/is-string/-/is-string-1.0.5.tgz", + "integrity": "sha512-buY6VNRjhQMiF1qWDouloZlQbRhDPCebwxSjxMjxgemYT46YMd2NR0/H+fBhEfWX4A/w9TBJ+ol+okqJKFE6vQ==", + "dev": true + }, + "is-symbol": { + "version": "1.0.3", + "resolved": "https://registry.npmjs.org/is-symbol/-/is-symbol-1.0.3.tgz", + "integrity": "sha512-OwijhaRSgqvhm/0ZdAcXNZt9lYdKFpcRDT5ULUuYXPoT794UNOdU+gpT6Rzo7b4V2HUl/op6GqY894AZwv9faQ==", + "dev": true, + "requires": { + "has-symbols": "^1.0.1" + } + }, "is-typedarray": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/is-typedarray/-/is-typedarray-1.0.0.tgz", @@ -1264,11 +2338,33 @@ "resolved": "https://registry.npmjs.org/isarray/-/isarray-1.0.0.tgz", "integrity": "sha512-VLghIWNM6ELQzo7zwmcg0NmTVyWKYjvIeM83yjp0wRDTmUnrM678fQbcKBo6n2CJEF0szoG//ytg+TKla89ALQ==" }, + "isexe": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/isexe/-/isexe-2.0.0.tgz", + "integrity": "sha512-RHxMLp9lnKHGHRng9QFhRCMbYAcVpn69smSGcq3f36xjgVVWThj4qqLbTLlq7Ssj8B+fIQ1EuCEGI2lKsyQeIw==", + "dev": true + }, "isstream": { "version": "0.1.2", "resolved": "https://registry.npmjs.org/isstream/-/isstream-0.1.2.tgz", "integrity": "sha512-Yljz7ffyPbrLpLngrMtZ7NduUgVvi6wG9RJ9IUcyCd59YQ911PBJphODUcbOVbqYfxe1wuYf/LJ8PauMRwsM/g==" }, + "js-tokens": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/js-tokens/-/js-tokens-4.0.0.tgz", + "integrity": "sha512-RdJUflcE3cUzKiMqQgsCu06FPu9UdIJO0beYbPhHN4k6apgJtifcoCtT9bcxOpYBtpD2kCM6Sbzg4CausW/PKQ==", + "dev": true + }, + "js-yaml": { + "version": "3.14.0", + "resolved": "https://registry.npmjs.org/js-yaml/-/js-yaml-3.14.0.tgz", + "integrity": "sha512-/4IbIeHcD9VMHFqDR/gQ7EdZdLimOvW2DdcxFjdyyZ9NsbS+ccrXqVWDtab/lRl5AlUqmpBx8EhPaWR+OtY17A==", + "dev": true, + "requires": { + "argparse": "^1.0.7", + "esprima": "^4.0.0" + } + }, "jsbn": { "version": "0.1.1", "resolved": "https://registry.npmjs.org/jsbn/-/jsbn-0.1.1.tgz", @@ -1292,11 +2388,34 @@ "resolved": "https://registry.npmjs.org/json-schema-traverse/-/json-schema-traverse-0.4.1.tgz", "integrity": "sha512-xbbCH5dCYU5T8LcEhhuh7HJ88HXuW3qsI3Y0zOZFKfZEHcpWiHU/Jxzk629Brsab/mMiHQti9wMP+845RPe3Vg==" }, + "json-stable-stringify-without-jsonify": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/json-stable-stringify-without-jsonify/-/json-stable-stringify-without-jsonify-1.0.1.tgz", + "integrity": "sha512-Bdboy+l7tA3OGW6FjyFHWkP5LuByj1Tk33Ljyq0axyzdk9//JSi2u3fP1QSmd1KNwq6VOKYGlAu87CisVir6Pw==", + "dev": true + }, "json-stringify-safe": { "version": "5.0.1", "resolved": "https://registry.npmjs.org/json-stringify-safe/-/json-stringify-safe-5.0.1.tgz", "integrity": "sha512-ZClg6AaYvamvYEE82d3Iyd3vSSIjQ+odgjaTzRuO3s7toCdFKczob2i0zCh7JE8kWn17yvAWhUVxvqGwUalsRA==" }, + "json5": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/json5/-/json5-1.0.1.tgz", + "integrity": "sha512-aKS4WQjPenRxiQsC93MNfjx+nbF4PAdYzmd/1JIj8HYzqfbu86beTuNgXDzPknWk0n0uARlyewZo4s++ES36Ow==", + "dev": true, + "requires": { + "minimist": "^1.2.0" + }, + "dependencies": { + "minimist": { + "version": "1.2.5", + "resolved": "https://registry.npmjs.org/minimist/-/minimist-1.2.5.tgz", + "integrity": "sha512-FM9nNUYrRBAELZQT3xeZQ7fmMOBg6nWNmJKTcgsJeaLstP/UODVpGsr5OhXhhXg6f+qtJ8uiZ+PUxkDWcgIXLw==", + "dev": true + } + } + }, "jsprim": { "version": "1.4.1", "resolved": "https://registry.npmjs.org/jsprim/-/jsprim-1.4.1.tgz", @@ -1327,6 +2446,46 @@ "safe-buffer": "^5.0.1" } }, + "levn": { + "version": "0.3.0", + "resolved": "https://registry.npmjs.org/levn/-/levn-0.3.0.tgz", + "integrity": "sha512-0OO4y2iOHix2W6ujICbKIaEQXvFQHue65vUG3pb5EUomzPI90z9hsA1VsO/dbIIpC53J8gxM9Q4Oho0jrCM/yA==", + "dev": true, + "requires": { + "prelude-ls": "~1.1.2", + "type-check": "~0.3.2" + } + }, + "load-json-file": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/load-json-file/-/load-json-file-2.0.0.tgz", + "integrity": "sha512-3p6ZOGNbiX4CdvEd1VcE6yi78UrGNpjHO33noGwHCnT/o2fyllJDepsm8+mFFv/DvtwFHht5HIHSyOy5a+ChVQ==", + "dev": true, + "requires": { + "graceful-fs": "^4.1.2", + "parse-json": "^2.2.0", + "pify": "^2.0.0", + "strip-bom": "^3.0.0" + }, + "dependencies": { + "pify": { + "version": "2.3.0", + "resolved": "https://registry.npmjs.org/pify/-/pify-2.3.0.tgz", + "integrity": "sha512-udgsAY+fTnvv7kI7aaxbqwWNb0AHiB0qBO89PZKPkoTmGOgdbrHDKD+0B2X4uTfJ/FT1R09r9gTsjUjNJotuog==", + "dev": true + } + } + }, + "locate-path": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/locate-path/-/locate-path-2.0.0.tgz", + "integrity": "sha512-NCI2kiDkyR7VeEKm27Kda/iQHyKJe1Bu0FlTbYp3CqJu+9IFe9bLyAjMxf5ZDDbEg+iMPzB5zYyUTSm8wVTKmA==", + "dev": true, + "requires": { + "p-locate": "^2.0.0", + "path-exists": "^3.0.0" + } + }, "lodash": { "version": "4.17.15", "resolved": "https://registry.npmjs.org/lodash/-/lodash-4.17.15.tgz", @@ -1342,11 +2501,29 @@ "resolved": "https://registry.npmjs.org/lodash.flatten/-/lodash.flatten-4.4.0.tgz", "integrity": "sha512-C5N2Z3DgnnKr0LOpv/hKCgKdb7ZZwafIrsesve6lmzvZIRZRGaZ/l6Q8+2W7NaT+ZwO3fFlSCzCzrDCFdJfZ4g==" }, + "lodash.memoize": { + "version": "4.1.2", + "resolved": "https://registry.npmjs.org/lodash.memoize/-/lodash.memoize-4.1.2.tgz", + "integrity": "sha512-t7j+NzmgnQzTAYXcsHYLgimltOV1MXHtlOWf6GjL9Kj8GK5FInw5JotxvbOs+IvV1/Dzo04/fCGfLVs7aXb4Ag==", + "dev": true + }, + "lodash.merge": { + "version": "4.6.2", + "resolved": "https://registry.npmjs.org/lodash.merge/-/lodash.merge-4.6.2.tgz", + "integrity": "sha512-0KpjqXRVvrYyCsX1swR/XTK0va6VQkQM6MNo7PqW77ByjAhoARA8EfrP1N4+KlKj8YS0ZUCtRT/YUuhyYDujIQ==", + "dev": true + }, "lodash.pickby": { "version": "4.6.0", "resolved": "https://registry.npmjs.org/lodash.pickby/-/lodash.pickby-4.6.0.tgz", "integrity": "sha512-AZV+GsS/6ckvPOVQPXSiFFacKvKB4kOQu6ynt9wz0F3LO4R9Ij4K1ddYsIytDpSgLz88JHd9P+oaLeej5/Sl7Q==" }, + "lodash.unescape": { + "version": "4.0.1", + "resolved": "https://registry.npmjs.org/lodash.unescape/-/lodash.unescape-4.0.1.tgz", + "integrity": "sha512-DhhGRshNS1aX6s5YdBE3njCCouPgnG29ebyHvImlZzXZf2SHgt+J08DHgytTPnpywNbO1Y8mNUFyQuIDBq2JZg==", + "dev": true + }, "logger-sharelatex": { "version": "1.7.0", "resolved": "https://registry.npmjs.org/logger-sharelatex/-/logger-sharelatex-1.7.0.tgz", @@ -1434,6 +2611,64 @@ } } }, + "loglevel": { + "version": "1.6.8", + "resolved": "https://registry.npmjs.org/loglevel/-/loglevel-1.6.8.tgz", + "integrity": "sha512-bsU7+gc9AJ2SqpzxwU3+1fedl8zAntbtC5XYlt3s2j1hJcn2PsXSmgN8TaLG/J1/2mod4+cE/3vNL70/c1RNCA==", + "dev": true + }, + "loglevel-colored-level-prefix": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/loglevel-colored-level-prefix/-/loglevel-colored-level-prefix-1.0.0.tgz", + "integrity": "sha512-u45Wcxxc+SdAlh4yeF/uKlC1SPUPCy0gullSNKXod5I4bmifzk+Q4lSLExNEVn19tGaJipbZ4V4jbFn79/6mVA==", + "dev": true, + "requires": { + "chalk": "^1.1.3", + "loglevel": "^1.4.1" + }, + "dependencies": { + "ansi-regex": { + "version": "2.1.1", + "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-2.1.1.tgz", + "integrity": "sha512-TIGnTpdo+E3+pCyAluZvtED5p5wCqLdezCyhPZzKPcxvFplEt4i+W7OONCKgeZFT3+y5NZZfOOS/Bdcanm1MYA==", + "dev": true + }, + "ansi-styles": { + "version": "2.2.1", + "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-2.2.1.tgz", + "integrity": "sha512-kmCevFghRiWM7HB5zTPULl4r9bVFSWjz62MhqizDGUrq2NWuNMQyuv4tHHoKJHs69M/MF64lEcHdYIocrdWQYA==", + "dev": true + }, + "chalk": { + "version": "1.1.3", + "resolved": "https://registry.npmjs.org/chalk/-/chalk-1.1.3.tgz", + "integrity": "sha512-U3lRVLMSlsCfjqYPbLyVv11M9CPW4I728d6TCKMAOJueEeB9/8o+eSsMnxPJD+Q+K909sdESg7C+tIkoH6on1A==", + "dev": true, + "requires": { + "ansi-styles": "^2.2.1", + "escape-string-regexp": "^1.0.2", + "has-ansi": "^2.0.0", + "strip-ansi": "^3.0.0", + "supports-color": "^2.0.0" + } + }, + "strip-ansi": { + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-3.0.1.tgz", + "integrity": "sha512-VhumSSbBqDTP8p2ZLKj40UjBCV4+v8bUSEpUb4KjRgWk9pbqGF4REFj6KEagidb2f/M6AzC0EmFyDNGaw9OCzg==", + "dev": true, + "requires": { + "ansi-regex": "^2.0.0" + } + }, + "supports-color": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-2.0.0.tgz", + "integrity": "sha512-KKNVtd6pCYgPIKU4cp2733HWYCpplQhddZLBUryaAHou723x+FRzQ5Df824Fj+IyyuiQTRoub4SnIFfIcrp70g==", + "dev": true + } + } + }, "lolex": { "version": "1.6.0", "resolved": "https://registry.npmjs.org/lolex/-/lolex-1.6.0.tgz", @@ -1467,6 +2702,30 @@ "statsd-parser": "~0.0.4" } }, + "make-plural": { + "version": "4.3.0", + "resolved": "https://registry.npmjs.org/make-plural/-/make-plural-4.3.0.tgz", + "integrity": "sha512-xTYd4JVHpSCW+aqDof6w/MebaMVNTVYBZhbB/vi513xXdiPT92JMVCo0Jq8W2UZnzYRFeVbQiQ+I25l13JuKvA==", + "dev": true, + "requires": { + "minimist": "^1.2.0" + }, + "dependencies": { + "minimist": { + "version": "1.2.5", + "resolved": "https://registry.npmjs.org/minimist/-/minimist-1.2.5.tgz", + "integrity": "sha512-FM9nNUYrRBAELZQT3xeZQ7fmMOBg6nWNmJKTcgsJeaLstP/UODVpGsr5OhXhhXg6f+qtJ8uiZ+PUxkDWcgIXLw==", + "dev": true, + "optional": true + } + } + }, + "map-obj": { + "version": "4.1.0", + "resolved": "https://registry.npmjs.org/map-obj/-/map-obj-4.1.0.tgz", + "integrity": "sha512-glc9y00wgtwcDmp7GaE/0b0OnxpNJsVf3ael/An6Fe2Q51LLwN1er6sdomLRzz5h0+yMpiYLhWYF5R7HeqVd4g==", + "dev": true + }, "media-typer": { "version": "0.3.0", "resolved": "https://registry.npmjs.org/media-typer/-/media-typer-0.3.0.tgz", @@ -1482,6 +2741,29 @@ "resolved": "https://registry.npmjs.org/mersenne/-/mersenne-0.0.4.tgz", "integrity": "sha512-XoSUL+nF8hMTKGQxUs8r3Btdsf1yuKKBdCCGbh3YXgCXuVKishpZv1CNc385w9s8t4Ynwc5h61BwW/FCVulkbg==" }, + "messageformat": { + "version": "2.3.0", + "resolved": "https://registry.npmjs.org/messageformat/-/messageformat-2.3.0.tgz", + "integrity": "sha512-uTzvsv0lTeQxYI2y1NPa1lItL5VRI8Gb93Y2K2ue5gBPyrbJxfDi/EYWxh2PKv5yO42AJeeqblS9MJSh/IEk4w==", + "dev": true, + "requires": { + "make-plural": "^4.3.0", + "messageformat-formatters": "^2.0.1", + "messageformat-parser": "^4.1.2" + } + }, + "messageformat-formatters": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/messageformat-formatters/-/messageformat-formatters-2.0.1.tgz", + "integrity": "sha512-E/lQRXhtHwGuiQjI7qxkLp8AHbMD5r2217XNe/SREbBlSawe0lOqsFb7rflZJmlQFSULNLIqlcjjsCPlB3m3Mg==", + "dev": true + }, + "messageformat-parser": { + "version": "4.1.3", + "resolved": "https://registry.npmjs.org/messageformat-parser/-/messageformat-parser-4.1.3.tgz", + "integrity": "sha512-2fU3XDCanRqeOCkn7R5zW5VQHWf+T3hH65SzuqRvjatBK7r4uyFa5mEX+k6F9Bd04LVM5G4/BHBTUJsOdW7uyg==", + "dev": true + }, "methods": { "version": "1.1.2", "resolved": "https://registry.npmjs.org/methods/-/methods-1.1.2.tgz", @@ -1527,6 +2809,12 @@ "mime-db": "~1.33.0" } }, + "mimic-fn": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/mimic-fn/-/mimic-fn-2.1.0.tgz", + "integrity": "sha512-OqbOk5oEQeAZ8WXWydlu9HJjz9WVdEIvamMCcXmuqUYjTknH/sqsWvhQ3vgwKFRR1HpjvNBKQ37nbJgYzGqGcg==", + "dev": true + }, "minimatch": { "version": "3.0.4", "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-3.0.4.tgz", @@ -1640,6 +2928,12 @@ "resolved": "https://registry.npmjs.org/ms/-/ms-2.0.0.tgz", "integrity": "sha512-Tpp60P6IUJDTuOq/5Z8cdskzJujfwqfOTkrwIwj7IRISpnkJnT6SyJ4PCPnGMoFjC9ddhal5KVIYtAt97ix05A==" }, + "mute-stream": { + "version": "0.0.8", + "resolved": "https://registry.npmjs.org/mute-stream/-/mute-stream-0.0.8.tgz", + "integrity": "sha512-nnbWWOkoWyUsTjKrhgD0dcz22mdkSnpYqbEjIm2nhwhuxlSkpywJmBo8h0ZqJdkp73mb90SssHkN4rsRaBAfAA==", + "dev": true + }, "mv": { "version": "2.1.1", "resolved": "https://registry.npmjs.org/mv/-/mv-2.1.1.tgz", @@ -1662,6 +2956,12 @@ "integrity": "sha512-zkVhZUA3y8mbz652WrL5x0fB0ehrBkulWT3TomAQ9iDtyXZvzKeEA6GPxAItBYeNYl5yngKRX612qHOhvMkDeg==", "dev": true }, + "natural-compare": { + "version": "1.4.0", + "resolved": "https://registry.npmjs.org/natural-compare/-/natural-compare-1.4.0.tgz", + "integrity": "sha512-OWND8ei3VtNC9h7V60qff3SVobHr996CTwgxubgyQYEpg290h9J0buyECNNJexkFm5sOajh5G116RYA1c8ZMSw==", + "dev": true + }, "ncp": { "version": "2.0.0", "resolved": "https://registry.npmjs.org/ncp/-/ncp-2.0.0.tgz", @@ -1673,6 +2973,12 @@ "resolved": "https://registry.npmjs.org/negotiator/-/negotiator-0.6.2.tgz", "integrity": "sha512-hZXc7K2e+PgeI1eDBe/10Ard4ekbfrrqG8Ep+8Jmf4JID2bNg7NvCPOZN+kfF574pFQI7mum2AUqDidoKqcTOw==" }, + "nice-try": { + "version": "1.0.5", + "resolved": "https://registry.npmjs.org/nice-try/-/nice-try-1.0.5.tgz", + "integrity": "sha512-1nh45deeb5olNY7eX82BkPO7SSxR5SSYJiPTrTdFUVYwAl8CKMA5N9PjTYkHiRjisVcxcQ1HXdLhx2qxxJzLNQ==", + "dev": true + }, "node-fetch": { "version": "2.6.0", "resolved": "https://registry.npmjs.org/node-fetch/-/node-fetch-2.6.0.tgz", @@ -1683,11 +2989,67 @@ "resolved": "https://registry.npmjs.org/node-forge/-/node-forge-0.8.5.tgz", "integrity": "sha512-vFMQIWt+J/7FLNyKouZ9TazT74PRV3wgv9UT4cRjC8BffxFbKXkgIWR42URCPSnHm/QDz6BOlb2Q0U4+VQT67Q==" }, + "normalize-package-data": { + "version": "2.5.0", + "resolved": "https://registry.npmjs.org/normalize-package-data/-/normalize-package-data-2.5.0.tgz", + "integrity": "sha512-/5CMN3T0R4XTj4DcGaexo+roZSdSFW/0AOOTROrjxzCG1wrWXEsGbRKevjlIL+ZDE4sZlJr5ED4YW0yqmkK+eA==", + "dev": true, + "requires": { + "hosted-git-info": "^2.1.4", + "resolve": "^1.10.0", + "semver": "2 || 3 || 4 || 5", + "validate-npm-package-license": "^3.0.1" + }, + "dependencies": { + "semver": { + "version": "5.7.1", + "resolved": "https://registry.npmjs.org/semver/-/semver-5.7.1.tgz", + "integrity": "sha512-sauaDf/PZdVgrLTNYHRtpXa1iRiKcaebiKQ1BJdpQlWH2lCvexQdX55snPFyK7QzpudqbCI0qXFfOasHdyNDGQ==", + "dev": true + } + } + }, "oauth-sign": { "version": "0.9.0", "resolved": "https://registry.npmjs.org/oauth-sign/-/oauth-sign-0.9.0.tgz", "integrity": "sha512-fexhUFFPTGV8ybAtSIGbV6gOkSv8UtRbDBnAyLQw4QPKkgNlsH2ByPGtMUqdWkos6YCRmAqViwgZrJc/mRDzZQ==" }, + "object-inspect": { + "version": "1.8.0", + "resolved": "https://registry.npmjs.org/object-inspect/-/object-inspect-1.8.0.tgz", + "integrity": "sha512-jLdtEOB112fORuypAyl/50VRVIBIdVQOSUUGQHzJ4xBSbit81zRarz7GThkEFZy1RceYrWYcPcBFPQwHyAc1gA==", + "dev": true + }, + "object-keys": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/object-keys/-/object-keys-1.1.1.tgz", + "integrity": "sha512-NuAESUOUMrlIXOfHKzD6bpPu3tYt3xvjNdRIQ+FeT0lNb4K8WR70CaDxhuNguS2XG+GjkyMwOzsN5ZktImfhLA==", + "dev": true + }, + "object.assign": { + "version": "4.1.0", + "resolved": "https://registry.npmjs.org/object.assign/-/object.assign-4.1.0.tgz", + "integrity": "sha512-exHJeq6kBKj58mqGyTQ9DFvrZC/eR6OwxzoM9YRoGBqrXYonaFyGiFMuc9VZrXf7DarreEwMpurG3dd+CNyW5w==", + "dev": true, + "requires": { + "define-properties": "^1.1.2", + "function-bind": "^1.1.1", + "has-symbols": "^1.0.0", + "object-keys": "^1.0.11" + } + }, + "object.values": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/object.values/-/object.values-1.1.1.tgz", + "integrity": "sha512-WTa54g2K8iu0kmS/us18jEmdv1a4Wi//BZ/DTVYEcH0XhLM5NYdpDHja3gt57VrZLcNAO2WGA+KpWsDBaHt6eA==", + "dev": true, + "requires": { + "define-properties": "^1.1.3", + "es-abstract": "^1.17.0-next.1", + "function-bind": "^1.1.1", + "has": "^1.0.3" + } + }, "on-finished": { "version": "2.3.0", "resolved": "https://registry.npmjs.org/on-finished/-/on-finished-2.3.0.tgz", @@ -1709,11 +3071,40 @@ "wrappy": "1" } }, + "onetime": { + "version": "5.1.0", + "resolved": "https://registry.npmjs.org/onetime/-/onetime-5.1.0.tgz", + "integrity": "sha512-5NcSkPHhwTVFIQN+TUqXoS5+dlElHXdpAWu9I0HP20YOtIi+aZ0Ct82jdlILDxjLEAWwvm+qj1m6aEtsDVmm6Q==", + "dev": true, + "requires": { + "mimic-fn": "^2.1.0" + } + }, + "optionator": { + "version": "0.8.3", + "resolved": "https://registry.npmjs.org/optionator/-/optionator-0.8.3.tgz", + "integrity": "sha512-+IW9pACdk3XWmmTXG8m3upGUJst5XRGzxMRjXzAuJ1XnIFNvfhjjIuYkDvysnPQ7qzqVzLt78BCruntqRhWQbA==", + "dev": true, + "requires": { + "deep-is": "~0.1.3", + "fast-levenshtein": "~2.0.6", + "levn": "~0.3.0", + "prelude-ls": "~1.1.2", + "type-check": "~0.3.2", + "word-wrap": "~1.2.3" + } + }, "options": { "version": "0.0.6", "resolved": "https://registry.npmjs.org/options/-/options-0.0.6.tgz", "integrity": "sha512-bOj3L1ypm++N+n7CEbbe473A414AB7z+amKYshRb//iuL3MpdDCLhPnw6aVTdKB9g5ZRVHIEp8eUln6L2NUStg==" }, + "os-tmpdir": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/os-tmpdir/-/os-tmpdir-1.0.2.tgz", + "integrity": "sha512-D2FR03Vir7FIu45XBY20mTb+/ZSWB00sjU9jdQXt83gDrI4Ztz5Fs7/yy74g2N5SVQY4xY1qDr4rNddwYRVX0g==", + "dev": true + }, "p-limit": { "version": "2.2.2", "resolved": "https://registry.npmjs.org/p-limit/-/p-limit-2.2.2.tgz", @@ -1722,16 +3113,60 @@ "p-try": "^2.0.0" } }, + "p-locate": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/p-locate/-/p-locate-2.0.0.tgz", + "integrity": "sha512-nQja7m7gSKuewoVRen45CtVfODR3crN3goVQ0DDZ9N3yHxgpkuBhZqsaiotSQRrADUrne346peY7kT3TSACykg==", + "dev": true, + "requires": { + "p-limit": "^1.1.0" + }, + "dependencies": { + "p-limit": { + "version": "1.3.0", + "resolved": "https://registry.npmjs.org/p-limit/-/p-limit-1.3.0.tgz", + "integrity": "sha512-vvcXsLAJ9Dr5rQOPk7toZQZJApBl2K4J6dANSsEuh6QI41JYcsS/qhTGa9ErIUUgK3WNQoJYvylxvjqmiqEA9Q==", + "dev": true, + "requires": { + "p-try": "^1.0.0" + } + }, + "p-try": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/p-try/-/p-try-1.0.0.tgz", + "integrity": "sha512-U1etNYuMJoIz3ZXSrrySFjsXQTWOx2/jdi86L+2pRvph/qMKL6sbcCYdH23fqsbm8TH2Gn0OybpT4eSFlCVHww==", + "dev": true + } + } + }, "p-try": { "version": "2.2.0", "resolved": "https://registry.npmjs.org/p-try/-/p-try-2.2.0.tgz", "integrity": "sha512-R4nPAVTAU0B9D35/Gk3uJf/7XYbQcyohSKdvAxIRSNghFl4e71hVoGnBNQz9cWaXxO2I10KTC+3jMdvvoKw6dQ==" }, + "parent-module": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/parent-module/-/parent-module-1.0.1.tgz", + "integrity": "sha512-GQ2EWRpQV8/o+Aw8YqtfZZPfNRWZYkbidE9k5rpl/hC3vtHHBfGm2Ifi6qWV+coDGkrUKZAxE3Lot5kcsRlh+g==", + "dev": true, + "requires": { + "callsites": "^3.0.0" + } + }, "parse-duration": { "version": "0.1.2", "resolved": "https://registry.npmjs.org/parse-duration/-/parse-duration-0.1.2.tgz", "integrity": "sha512-0qfMZyjOUFBeEIvJ5EayfXJqaEXxQ+Oj2b7tWJM3hvEXvXsYCk05EDVI23oYnEw2NaFYUWdABEVPBvBMh8L/pA==" }, + "parse-json": { + "version": "2.2.0", + "resolved": "https://registry.npmjs.org/parse-json/-/parse-json-2.2.0.tgz", + "integrity": "sha512-QR/GGaKCkhwk1ePQNYDRKYZ3mwU9ypsKhB0XyFnLQdomyEqk3e8wpW3V5Jp88zbxK4n5ST1nqo+g9juTpownhQ==", + "dev": true, + "requires": { + "error-ex": "^1.2.0" + } + }, "parse-ms": { "version": "2.1.0", "resolved": "https://registry.npmjs.org/parse-ms/-/parse-ms-2.1.0.tgz", @@ -1742,11 +3177,29 @@ "resolved": "https://registry.npmjs.org/parseurl/-/parseurl-1.3.3.tgz", "integrity": "sha512-CiyeOxFT/JZyN5m0z9PfXw4SCBJ6Sygz1Dpl0wqjlhDEGGBP1GnsUVEL0p63hoG1fcj3fHynXi9NYO4nWOL+qQ==" }, + "path-exists": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/path-exists/-/path-exists-3.0.0.tgz", + "integrity": "sha512-bpC7GYwiDYQ4wYLe+FA8lhRjhQCMcQGuSgGGqDkg/QerRWw9CmGRT0iSOVRSZJ29NMLZgIzqaljJ63oaL4NIJQ==", + "dev": true + }, "path-is-absolute": { "version": "1.0.1", "resolved": "https://registry.npmjs.org/path-is-absolute/-/path-is-absolute-1.0.1.tgz", "integrity": "sha512-AVbw3UJ2e9bq64vSaS9Am0fje1Pa8pbGqTTsmXfaIiMpnr5DlDhfJOuLj9Sf95ZPVDAUerDfEk88MPmPe7UCQg==" }, + "path-is-inside": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/path-is-inside/-/path-is-inside-1.0.2.tgz", + "integrity": "sha512-DUWJr3+ULp4zXmol/SZkFf3JGsS9/SIv+Y3Rt93/UjPpDpklB5f1er4O3POIbUuUJ3FXgqte2Q7SrU6zAqwk8w==", + "dev": true + }, + "path-key": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/path-key/-/path-key-2.0.1.tgz", + "integrity": "sha512-fEHGKCSmUSDPv4uoj8AlD+joPlq3peND+HRYyxFz4KPw4z926S/b8rIuFs2FYJg3BwsxJf6A9/3eIdLaYC+9Dw==", + "dev": true + }, "path-parse": { "version": "1.0.6", "resolved": "https://registry.npmjs.org/path-parse/-/path-parse-1.0.6.tgz", @@ -1757,6 +3210,23 @@ "resolved": "https://registry.npmjs.org/path-to-regexp/-/path-to-regexp-0.1.7.tgz", "integrity": "sha512-5DFkuoqlv1uYQKxy8omFBeJPQcdoE07Kv2sferDCrAq1ohOU+MSDswDIbnx3YAM60qIOnYa53wBhXW0EbMonrQ==" }, + "path-type": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/path-type/-/path-type-2.0.0.tgz", + "integrity": "sha512-dUnb5dXUf+kzhC/W/F4e5/SkluXIFf5VUHolW1Eg1irn1hGWjPGdsRcvYJ1nD6lhk8Ir7VM0bHJKsYTx8Jx9OQ==", + "dev": true, + "requires": { + "pify": "^2.0.0" + }, + "dependencies": { + "pify": { + "version": "2.3.0", + "resolved": "https://registry.npmjs.org/pify/-/pify-2.3.0.tgz", + "integrity": "sha512-udgsAY+fTnvv7kI7aaxbqwWNb0AHiB0qBO89PZKPkoTmGOgdbrHDKD+0B2X4uTfJ/FT1R09r9gTsjUjNJotuog==", + "dev": true + } + } + }, "performance-now": { "version": "2.1.0", "resolved": "https://registry.npmjs.org/performance-now/-/performance-now-2.1.0.tgz", @@ -1767,11 +3237,641 @@ "resolved": "https://registry.npmjs.org/pify/-/pify-4.0.1.tgz", "integrity": "sha512-uB80kBFb/tfd68bVleG9T5GGsGPjJrLAUpR5PZIrhBnIaRTQRjqdJSsIKkOP6OAIFbj7GOrcudc5pNjZ+geV2g==" }, + "pkg-dir": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/pkg-dir/-/pkg-dir-2.0.0.tgz", + "integrity": "sha512-ojakdnUgL5pzJYWw2AIDEupaQCX5OPbM688ZevubICjdIX01PRSYKqm33fJoCOJBRseYCTUlQRnBNX+Pchaejw==", + "dev": true, + "requires": { + "find-up": "^2.1.0" + } + }, "policyfile": { "version": "0.0.4", "resolved": "https://registry.npmjs.org/policyfile/-/policyfile-0.0.4.tgz", "integrity": "sha512-UfDtlscNialXfmVEwEPm0t/5qtM0xPK025eYWd/ilv89hxLIhVQmt3QIzMHincLO2MBtZyww0386pt13J4aIhQ==" }, + "prelude-ls": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/prelude-ls/-/prelude-ls-1.1.2.tgz", + "integrity": "sha512-ESF23V4SKG6lVSGZgYNpbsiaAkdab6ZgOxe52p7+Kid3W3u3bxR4Vfd/o21dmN7jSt0IwgZ4v5MUd26FEtXE9w==", + "dev": true + }, + "prettier": { + "version": "2.0.5", + "resolved": "https://registry.npmjs.org/prettier/-/prettier-2.0.5.tgz", + "integrity": "sha512-7PtVymN48hGcO4fGjybyBSIWDsLU4H4XlvOHfq91pz9kkGlonzwTfYkaIEwiRg/dAJF9YlbsduBAgtYLi+8cFg==", + "dev": true + }, + "prettier-eslint": { + "version": "9.0.2", + "resolved": "https://registry.npmjs.org/prettier-eslint/-/prettier-eslint-9.0.2.tgz", + "integrity": "sha512-u6EQqxUhaGfra9gy9shcR7MT7r/2twwEfRGy1tfzyaJvLQwSg34M9IU5HuF7FsLW2QUgr5VIUc56EPWibw1pdw==", + "dev": true, + "requires": { + "@typescript-eslint/parser": "^1.10.2", + "common-tags": "^1.4.0", + "core-js": "^3.1.4", + "dlv": "^1.1.0", + "eslint": "^5.0.0", + "indent-string": "^4.0.0", + "lodash.merge": "^4.6.0", + "loglevel-colored-level-prefix": "^1.0.0", + "prettier": "^1.7.0", + "pretty-format": "^23.0.1", + "require-relative": "^0.8.7", + "typescript": "^3.2.1", + "vue-eslint-parser": "^2.0.2" + }, + "dependencies": { + "ansi-escapes": { + "version": "3.2.0", + "resolved": "https://registry.npmjs.org/ansi-escapes/-/ansi-escapes-3.2.0.tgz", + "integrity": "sha512-cBhpre4ma+U0T1oM5fXg7Dy1Jw7zzwv7lt/GoCpr+hDQJoYnKVPLL4dCvSEFMmQurOQvSrwT7SL/DAlhBI97RQ==", + "dev": true + }, + "ansi-regex": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-3.0.0.tgz", + "integrity": "sha512-wFUFA5bg5dviipbQQ32yOQhl6gcJaJXiHE7dvR8VYPG97+J/GNC5FKGepKdEDUFeXRzDxPF1X/Btc8L+v7oqIQ==", + "dev": true + }, + "cli-cursor": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/cli-cursor/-/cli-cursor-2.1.0.tgz", + "integrity": "sha512-8lgKz8LmCRYZZQDpRyT2m5rKJ08TnU4tR9FFFW2rxpxR1FzWi4PQ/NfyODchAatHaUgnSPVcx/R5w6NuTBzFiw==", + "dev": true, + "requires": { + "restore-cursor": "^2.0.0" + } + }, + "debug": { + "version": "4.1.1", + "resolved": "https://registry.npmjs.org/debug/-/debug-4.1.1.tgz", + "integrity": "sha512-pYAIzeRo8J6KPEaJ0VWOh5Pzkbw/RetuzehGM7QRRX5he4fPHx2rdKMB256ehJCkX+XRQm16eZLqLNS8RSZXZw==", + "dev": true, + "requires": { + "ms": "^2.1.1" + } + }, + "eslint": { + "version": "5.16.0", + "resolved": "https://registry.npmjs.org/eslint/-/eslint-5.16.0.tgz", + "integrity": "sha512-S3Rz11i7c8AA5JPv7xAH+dOyq/Cu/VXHiHXBPOU1k/JAM5dXqQPt3qcrhpHSorXmrpu2g0gkIBVXAqCpzfoZIg==", + "dev": true, + "requires": { + "@babel/code-frame": "^7.0.0", + "ajv": "^6.9.1", + "chalk": "^2.1.0", + "cross-spawn": "^6.0.5", + "debug": "^4.0.1", + "doctrine": "^3.0.0", + "eslint-scope": "^4.0.3", + "eslint-utils": "^1.3.1", + "eslint-visitor-keys": "^1.0.0", + "espree": "^5.0.1", + "esquery": "^1.0.1", + "esutils": "^2.0.2", + "file-entry-cache": "^5.0.1", + "functional-red-black-tree": "^1.0.1", + "glob": "^7.1.2", + "globals": "^11.7.0", + "ignore": "^4.0.6", + "import-fresh": "^3.0.0", + "imurmurhash": "^0.1.4", + "inquirer": "^6.2.2", + "js-yaml": "^3.13.0", + "json-stable-stringify-without-jsonify": "^1.0.1", + "levn": "^0.3.0", + "lodash": "^4.17.11", + "minimatch": "^3.0.4", + "mkdirp": "^0.5.1", + "natural-compare": "^1.4.0", + "optionator": "^0.8.2", + "path-is-inside": "^1.0.2", + "progress": "^2.0.0", + "regexpp": "^2.0.1", + "semver": "^5.5.1", + "strip-ansi": "^4.0.0", + "strip-json-comments": "^2.0.1", + "table": "^5.2.3", + "text-table": "^0.2.0" + } + }, + "eslint-scope": { + "version": "4.0.3", + "resolved": "https://registry.npmjs.org/eslint-scope/-/eslint-scope-4.0.3.tgz", + "integrity": "sha512-p7VutNr1O/QrxysMo3E45FjYDTeXBy0iTltPFNSqKAIfjDSXC+4dj+qfyuD8bfAXrW/y6lW3O76VaYNPKfpKrg==", + "dev": true, + "requires": { + "esrecurse": "^4.1.0", + "estraverse": "^4.1.1" + } + }, + "espree": { + "version": "5.0.1", + "resolved": "https://registry.npmjs.org/espree/-/espree-5.0.1.tgz", + "integrity": "sha512-qWAZcWh4XE/RwzLJejfcofscgMc9CamR6Tn1+XRXNzrvUSSbiAjGOI/fggztjIi7y9VLPqnICMIPiGyr8JaZ0A==", + "dev": true, + "requires": { + "acorn": "^6.0.7", + "acorn-jsx": "^5.0.0", + "eslint-visitor-keys": "^1.0.0" + } + }, + "figures": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/figures/-/figures-2.0.0.tgz", + "integrity": "sha512-Oa2M9atig69ZkfwiApY8F2Yy+tzMbazyvqv21R0NsSC8floSOC09BbT1ITWAdoMGQvJ/aZnR1KMwdx9tvHnTNA==", + "dev": true, + "requires": { + "escape-string-regexp": "^1.0.5" + } + }, + "glob": { + "version": "7.1.6", + "resolved": "https://registry.npmjs.org/glob/-/glob-7.1.6.tgz", + "integrity": "sha512-LwaxwyZ72Lk7vZINtNNrywX0ZuLyStrdDtabefZKAY5ZGJhVtgdznluResxNmPitE0SAO+O26sWTHeKSI2wMBA==", + "dev": true, + "requires": { + "fs.realpath": "^1.0.0", + "inflight": "^1.0.4", + "inherits": "2", + "minimatch": "^3.0.4", + "once": "^1.3.0", + "path-is-absolute": "^1.0.0" + } + }, + "globals": { + "version": "11.12.0", + "resolved": "https://registry.npmjs.org/globals/-/globals-11.12.0.tgz", + "integrity": "sha512-WOBp/EEGUiIsJSp7wcv/y6MO+lV9UoncWqxuFfm8eBwzWNgyfBd6Gz+IeKQ9jCmyhoH99g15M3T+QaVHFjizVA==", + "dev": true + }, + "inquirer": { + "version": "6.5.2", + "resolved": "https://registry.npmjs.org/inquirer/-/inquirer-6.5.2.tgz", + "integrity": "sha512-cntlB5ghuB0iuO65Ovoi8ogLHiWGs/5yNrtUcKjFhSSiVeAIVpD7koaSU9RM8mpXw5YDi9RdYXGQMaOURB7ycQ==", + "dev": true, + "requires": { + "ansi-escapes": "^3.2.0", + "chalk": "^2.4.2", + "cli-cursor": "^2.1.0", + "cli-width": "^2.0.0", + "external-editor": "^3.0.3", + "figures": "^2.0.0", + "lodash": "^4.17.12", + "mute-stream": "0.0.7", + "run-async": "^2.2.0", + "rxjs": "^6.4.0", + "string-width": "^2.1.0", + "strip-ansi": "^5.1.0", + "through": "^2.3.6" + }, + "dependencies": { + "ansi-regex": { + "version": "4.1.0", + "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-4.1.0.tgz", + "integrity": "sha512-1apePfXM1UOSqw0o9IiFAovVz9M5S1Dg+4TrDwfMewQ6p/rmMueb7tWZjQ1rx4Loy1ArBggoqGpfqqdI4rondg==", + "dev": true + }, + "strip-ansi": { + "version": "5.2.0", + "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-5.2.0.tgz", + "integrity": "sha512-DuRs1gKbBqsMKIZlrffwlug8MHkcnpjs5VPmL1PAh+mA30U0DTotfDZ0d2UUsXpPmPmMMJ6W773MaA3J+lbiWA==", + "dev": true, + "requires": { + "ansi-regex": "^4.1.0" + } + } + } + }, + "is-fullwidth-code-point": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/is-fullwidth-code-point/-/is-fullwidth-code-point-2.0.0.tgz", + "integrity": "sha512-VHskAKYM8RfSFXwee5t5cbN5PZeq1Wrh6qd5bkyiXIf6UQcN6w/A0eXM9r6t8d+GYOh+o6ZhiEnb88LN/Y8m2w==", + "dev": true + }, + "mimic-fn": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/mimic-fn/-/mimic-fn-1.2.0.tgz", + "integrity": "sha512-jf84uxzwiuiIVKiOLpfYk7N46TSy8ubTonmneY9vrpHNAnp0QBt2BxWV9dO3/j+BoVAb+a5G6YDPW3M5HOdMWQ==", + "dev": true + }, + "ms": { + "version": "2.1.2", + "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.2.tgz", + "integrity": "sha512-sGkPx+VjMtmA6MX27oA4FBFELFCZZ4S4XqeGOXCv68tT+jb3vk/RyaKWP0PTKyWtmLSM0b+adUTEvbs1PEaH2w==", + "dev": true + }, + "mute-stream": { + "version": "0.0.7", + "resolved": "https://registry.npmjs.org/mute-stream/-/mute-stream-0.0.7.tgz", + "integrity": "sha512-r65nCZhrbXXb6dXOACihYApHw2Q6pV0M3V0PSxd74N0+D8nzAdEAITq2oAjA1jVnKI+tGvEBUpqiMh0+rW6zDQ==", + "dev": true + }, + "onetime": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/onetime/-/onetime-2.0.1.tgz", + "integrity": "sha512-oyyPpiMaKARvvcgip+JV+7zci5L8D1W9RZIz2l1o08AM3pfspitVWnPt3mzHcBPp12oYMTy0pqrFs/C+m3EwsQ==", + "dev": true, + "requires": { + "mimic-fn": "^1.0.0" + } + }, + "prettier": { + "version": "1.19.1", + "resolved": "https://registry.npmjs.org/prettier/-/prettier-1.19.1.tgz", + "integrity": "sha512-s7PoyDv/II1ObgQunCbB9PdLmUcBZcnWOcxDh7O0N/UwDEsHyqkW+Qh28jW+mVuCdx7gLB0BotYI1Y6uI9iyew==", + "dev": true + }, + "restore-cursor": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/restore-cursor/-/restore-cursor-2.0.0.tgz", + "integrity": "sha512-6IzJLuGi4+R14vwagDHX+JrXmPVtPpn4mffDJ1UdR7/Edm87fl6yi8mMBIVvFtJaNTUvjughmW4hwLhRG7gC1Q==", + "dev": true, + "requires": { + "onetime": "^2.0.0", + "signal-exit": "^3.0.2" + } + }, + "semver": { + "version": "5.7.1", + "resolved": "https://registry.npmjs.org/semver/-/semver-5.7.1.tgz", + "integrity": "sha512-sauaDf/PZdVgrLTNYHRtpXa1iRiKcaebiKQ1BJdpQlWH2lCvexQdX55snPFyK7QzpudqbCI0qXFfOasHdyNDGQ==", + "dev": true + }, + "string-width": { + "version": "2.1.1", + "resolved": "https://registry.npmjs.org/string-width/-/string-width-2.1.1.tgz", + "integrity": "sha512-nOqH59deCq9SRHlxq1Aw85Jnt4w6KvLKqWVik6oA9ZklXLNIOlqg4F2yrT1MVaTjAqvVwdfeZ7w7aCvJD7ugkw==", + "dev": true, + "requires": { + "is-fullwidth-code-point": "^2.0.0", + "strip-ansi": "^4.0.0" + } + }, + "strip-ansi": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-4.0.0.tgz", + "integrity": "sha512-4XaJ2zQdCzROZDivEVIDPkcQn8LMFSa8kj8Gxb/Lnwzv9A8VctNZ+lfivC/sV3ivW8ElJTERXZoPBRrZKkNKow==", + "dev": true, + "requires": { + "ansi-regex": "^3.0.0" + } + }, + "strip-json-comments": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/strip-json-comments/-/strip-json-comments-2.0.1.tgz", + "integrity": "sha512-4gB8na07fecVVkOI6Rs4e7T6NOTki5EmL7TUduTs6bu3EdnSycntVJ4re8kgZA+wx9IueI2Y11bfbgwtzuE0KQ==", + "dev": true + } + } + }, + "prettier-eslint-cli": { + "version": "5.0.0", + "resolved": "https://registry.npmjs.org/prettier-eslint-cli/-/prettier-eslint-cli-5.0.0.tgz", + "integrity": "sha512-cei9UbN1aTrz3sQs88CWpvY/10PYTevzd76zoG1tdJ164OhmNTFRKPTOZrutVvscoQWzbnLKkviS3gu5JXwvZg==", + "dev": true, + "requires": { + "arrify": "^2.0.1", + "boolify": "^1.0.0", + "camelcase-keys": "^6.0.0", + "chalk": "^2.4.2", + "common-tags": "^1.8.0", + "core-js": "^3.1.4", + "eslint": "^5.0.0", + "find-up": "^4.1.0", + "get-stdin": "^7.0.0", + "glob": "^7.1.4", + "ignore": "^5.1.2", + "lodash.memoize": "^4.1.2", + "loglevel-colored-level-prefix": "^1.0.0", + "messageformat": "^2.2.1", + "prettier-eslint": "^9.0.0", + "rxjs": "^6.5.2", + "yargs": "^13.2.4" + }, + "dependencies": { + "ansi-escapes": { + "version": "3.2.0", + "resolved": "https://registry.npmjs.org/ansi-escapes/-/ansi-escapes-3.2.0.tgz", + "integrity": "sha512-cBhpre4ma+U0T1oM5fXg7Dy1Jw7zzwv7lt/GoCpr+hDQJoYnKVPLL4dCvSEFMmQurOQvSrwT7SL/DAlhBI97RQ==", + "dev": true + }, + "ansi-regex": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-3.0.0.tgz", + "integrity": "sha512-wFUFA5bg5dviipbQQ32yOQhl6gcJaJXiHE7dvR8VYPG97+J/GNC5FKGepKdEDUFeXRzDxPF1X/Btc8L+v7oqIQ==", + "dev": true + }, + "cli-cursor": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/cli-cursor/-/cli-cursor-2.1.0.tgz", + "integrity": "sha512-8lgKz8LmCRYZZQDpRyT2m5rKJ08TnU4tR9FFFW2rxpxR1FzWi4PQ/NfyODchAatHaUgnSPVcx/R5w6NuTBzFiw==", + "dev": true, + "requires": { + "restore-cursor": "^2.0.0" + } + }, + "debug": { + "version": "4.1.1", + "resolved": "https://registry.npmjs.org/debug/-/debug-4.1.1.tgz", + "integrity": "sha512-pYAIzeRo8J6KPEaJ0VWOh5Pzkbw/RetuzehGM7QRRX5he4fPHx2rdKMB256ehJCkX+XRQm16eZLqLNS8RSZXZw==", + "dev": true, + "requires": { + "ms": "^2.1.1" + } + }, + "eslint": { + "version": "5.16.0", + "resolved": "https://registry.npmjs.org/eslint/-/eslint-5.16.0.tgz", + "integrity": "sha512-S3Rz11i7c8AA5JPv7xAH+dOyq/Cu/VXHiHXBPOU1k/JAM5dXqQPt3qcrhpHSorXmrpu2g0gkIBVXAqCpzfoZIg==", + "dev": true, + "requires": { + "@babel/code-frame": "^7.0.0", + "ajv": "^6.9.1", + "chalk": "^2.1.0", + "cross-spawn": "^6.0.5", + "debug": "^4.0.1", + "doctrine": "^3.0.0", + "eslint-scope": "^4.0.3", + "eslint-utils": "^1.3.1", + "eslint-visitor-keys": "^1.0.0", + "espree": "^5.0.1", + "esquery": "^1.0.1", + "esutils": "^2.0.2", + "file-entry-cache": "^5.0.1", + "functional-red-black-tree": "^1.0.1", + "glob": "^7.1.2", + "globals": "^11.7.0", + "ignore": "^4.0.6", + "import-fresh": "^3.0.0", + "imurmurhash": "^0.1.4", + "inquirer": "^6.2.2", + "js-yaml": "^3.13.0", + "json-stable-stringify-without-jsonify": "^1.0.1", + "levn": "^0.3.0", + "lodash": "^4.17.11", + "minimatch": "^3.0.4", + "mkdirp": "^0.5.1", + "natural-compare": "^1.4.0", + "optionator": "^0.8.2", + "path-is-inside": "^1.0.2", + "progress": "^2.0.0", + "regexpp": "^2.0.1", + "semver": "^5.5.1", + "strip-ansi": "^4.0.0", + "strip-json-comments": "^2.0.1", + "table": "^5.2.3", + "text-table": "^0.2.0" + }, + "dependencies": { + "ignore": { + "version": "4.0.6", + "resolved": "https://registry.npmjs.org/ignore/-/ignore-4.0.6.tgz", + "integrity": "sha512-cyFDKrqc/YdcWFniJhzI42+AzS+gNwmUzOSFcRCQYwySuBBBy/KjuxWLZ/FHEH6Moq1NizMOBWyTcv8O4OZIMg==", + "dev": true + } + } + }, + "eslint-scope": { + "version": "4.0.3", + "resolved": "https://registry.npmjs.org/eslint-scope/-/eslint-scope-4.0.3.tgz", + "integrity": "sha512-p7VutNr1O/QrxysMo3E45FjYDTeXBy0iTltPFNSqKAIfjDSXC+4dj+qfyuD8bfAXrW/y6lW3O76VaYNPKfpKrg==", + "dev": true, + "requires": { + "esrecurse": "^4.1.0", + "estraverse": "^4.1.1" + } + }, + "espree": { + "version": "5.0.1", + "resolved": "https://registry.npmjs.org/espree/-/espree-5.0.1.tgz", + "integrity": "sha512-qWAZcWh4XE/RwzLJejfcofscgMc9CamR6Tn1+XRXNzrvUSSbiAjGOI/fggztjIi7y9VLPqnICMIPiGyr8JaZ0A==", + "dev": true, + "requires": { + "acorn": "^6.0.7", + "acorn-jsx": "^5.0.0", + "eslint-visitor-keys": "^1.0.0" + } + }, + "figures": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/figures/-/figures-2.0.0.tgz", + "integrity": "sha512-Oa2M9atig69ZkfwiApY8F2Yy+tzMbazyvqv21R0NsSC8floSOC09BbT1ITWAdoMGQvJ/aZnR1KMwdx9tvHnTNA==", + "dev": true, + "requires": { + "escape-string-regexp": "^1.0.5" + } + }, + "find-up": { + "version": "4.1.0", + "resolved": "https://registry.npmjs.org/find-up/-/find-up-4.1.0.tgz", + "integrity": "sha512-PpOwAdQ/YlXQ2vj8a3h8IipDuYRi3wceVQQGYWxNINccq40Anw7BlsEXCMbt1Zt+OLA6Fq9suIpIWD0OsnISlw==", + "dev": true, + "requires": { + "locate-path": "^5.0.0", + "path-exists": "^4.0.0" + } + }, + "get-stdin": { + "version": "7.0.0", + "resolved": "https://registry.npmjs.org/get-stdin/-/get-stdin-7.0.0.tgz", + "integrity": "sha512-zRKcywvrXlXsA0v0i9Io4KDRaAw7+a1ZpjRwl9Wox8PFlVCCHra7E9c4kqXCoCM9nR5tBkaTTZRBoCm60bFqTQ==", + "dev": true + }, + "glob": { + "version": "7.1.6", + "resolved": "https://registry.npmjs.org/glob/-/glob-7.1.6.tgz", + "integrity": "sha512-LwaxwyZ72Lk7vZINtNNrywX0ZuLyStrdDtabefZKAY5ZGJhVtgdznluResxNmPitE0SAO+O26sWTHeKSI2wMBA==", + "dev": true, + "requires": { + "fs.realpath": "^1.0.0", + "inflight": "^1.0.4", + "inherits": "2", + "minimatch": "^3.0.4", + "once": "^1.3.0", + "path-is-absolute": "^1.0.0" + } + }, + "globals": { + "version": "11.12.0", + "resolved": "https://registry.npmjs.org/globals/-/globals-11.12.0.tgz", + "integrity": "sha512-WOBp/EEGUiIsJSp7wcv/y6MO+lV9UoncWqxuFfm8eBwzWNgyfBd6Gz+IeKQ9jCmyhoH99g15M3T+QaVHFjizVA==", + "dev": true + }, + "ignore": { + "version": "5.1.8", + "resolved": "https://registry.npmjs.org/ignore/-/ignore-5.1.8.tgz", + "integrity": "sha512-BMpfD7PpiETpBl/A6S498BaIJ6Y/ABT93ETbby2fP00v4EbvPBXWEoaR1UBPKs3iR53pJY7EtZk5KACI57i1Uw==", + "dev": true + }, + "inquirer": { + "version": "6.5.2", + "resolved": "https://registry.npmjs.org/inquirer/-/inquirer-6.5.2.tgz", + "integrity": "sha512-cntlB5ghuB0iuO65Ovoi8ogLHiWGs/5yNrtUcKjFhSSiVeAIVpD7koaSU9RM8mpXw5YDi9RdYXGQMaOURB7ycQ==", + "dev": true, + "requires": { + "ansi-escapes": "^3.2.0", + "chalk": "^2.4.2", + "cli-cursor": "^2.1.0", + "cli-width": "^2.0.0", + "external-editor": "^3.0.3", + "figures": "^2.0.0", + "lodash": "^4.17.12", + "mute-stream": "0.0.7", + "run-async": "^2.2.0", + "rxjs": "^6.4.0", + "string-width": "^2.1.0", + "strip-ansi": "^5.1.0", + "through": "^2.3.6" + }, + "dependencies": { + "ansi-regex": { + "version": "4.1.0", + "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-4.1.0.tgz", + "integrity": "sha512-1apePfXM1UOSqw0o9IiFAovVz9M5S1Dg+4TrDwfMewQ6p/rmMueb7tWZjQ1rx4Loy1ArBggoqGpfqqdI4rondg==", + "dev": true + }, + "strip-ansi": { + "version": "5.2.0", + "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-5.2.0.tgz", + "integrity": "sha512-DuRs1gKbBqsMKIZlrffwlug8MHkcnpjs5VPmL1PAh+mA30U0DTotfDZ0d2UUsXpPmPmMMJ6W773MaA3J+lbiWA==", + "dev": true, + "requires": { + "ansi-regex": "^4.1.0" + } + } + } + }, + "is-fullwidth-code-point": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/is-fullwidth-code-point/-/is-fullwidth-code-point-2.0.0.tgz", + "integrity": "sha512-VHskAKYM8RfSFXwee5t5cbN5PZeq1Wrh6qd5bkyiXIf6UQcN6w/A0eXM9r6t8d+GYOh+o6ZhiEnb88LN/Y8m2w==", + "dev": true + }, + "locate-path": { + "version": "5.0.0", + "resolved": "https://registry.npmjs.org/locate-path/-/locate-path-5.0.0.tgz", + "integrity": "sha512-t7hw9pI+WvuwNJXwk5zVHpyhIqzg2qTlklJOf0mVxGSbe3Fp2VieZcduNYjaLDoy6p9uGpQEGWG87WpMKlNq8g==", + "dev": true, + "requires": { + "p-locate": "^4.1.0" + } + }, + "mimic-fn": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/mimic-fn/-/mimic-fn-1.2.0.tgz", + "integrity": "sha512-jf84uxzwiuiIVKiOLpfYk7N46TSy8ubTonmneY9vrpHNAnp0QBt2BxWV9dO3/j+BoVAb+a5G6YDPW3M5HOdMWQ==", + "dev": true + }, + "ms": { + "version": "2.1.2", + "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.2.tgz", + "integrity": "sha512-sGkPx+VjMtmA6MX27oA4FBFELFCZZ4S4XqeGOXCv68tT+jb3vk/RyaKWP0PTKyWtmLSM0b+adUTEvbs1PEaH2w==", + "dev": true + }, + "mute-stream": { + "version": "0.0.7", + "resolved": "https://registry.npmjs.org/mute-stream/-/mute-stream-0.0.7.tgz", + "integrity": "sha512-r65nCZhrbXXb6dXOACihYApHw2Q6pV0M3V0PSxd74N0+D8nzAdEAITq2oAjA1jVnKI+tGvEBUpqiMh0+rW6zDQ==", + "dev": true + }, + "onetime": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/onetime/-/onetime-2.0.1.tgz", + "integrity": "sha512-oyyPpiMaKARvvcgip+JV+7zci5L8D1W9RZIz2l1o08AM3pfspitVWnPt3mzHcBPp12oYMTy0pqrFs/C+m3EwsQ==", + "dev": true, + "requires": { + "mimic-fn": "^1.0.0" + } + }, + "p-locate": { + "version": "4.1.0", + "resolved": "https://registry.npmjs.org/p-locate/-/p-locate-4.1.0.tgz", + "integrity": "sha512-R79ZZ/0wAxKGu3oYMlz8jy/kbhsNrS7SKZ7PxEHBgJ5+F2mtFW2fK2cOtBh1cHYkQsbzFV7I+EoRKe6Yt0oK7A==", + "dev": true, + "requires": { + "p-limit": "^2.2.0" + } + }, + "path-exists": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/path-exists/-/path-exists-4.0.0.tgz", + "integrity": "sha512-ak9Qy5Q7jYb2Wwcey5Fpvg2KoAc/ZIhLSLOSBmRmygPsGwkVVt0fZa0qrtMz+m6tJTAHfZQ8FnmB4MG4LWy7/w==", + "dev": true + }, + "restore-cursor": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/restore-cursor/-/restore-cursor-2.0.0.tgz", + "integrity": "sha512-6IzJLuGi4+R14vwagDHX+JrXmPVtPpn4mffDJ1UdR7/Edm87fl6yi8mMBIVvFtJaNTUvjughmW4hwLhRG7gC1Q==", + "dev": true, + "requires": { + "onetime": "^2.0.0", + "signal-exit": "^3.0.2" + } + }, + "semver": { + "version": "5.7.1", + "resolved": "https://registry.npmjs.org/semver/-/semver-5.7.1.tgz", + "integrity": "sha512-sauaDf/PZdVgrLTNYHRtpXa1iRiKcaebiKQ1BJdpQlWH2lCvexQdX55snPFyK7QzpudqbCI0qXFfOasHdyNDGQ==", + "dev": true + }, + "string-width": { + "version": "2.1.1", + "resolved": "https://registry.npmjs.org/string-width/-/string-width-2.1.1.tgz", + "integrity": "sha512-nOqH59deCq9SRHlxq1Aw85Jnt4w6KvLKqWVik6oA9ZklXLNIOlqg4F2yrT1MVaTjAqvVwdfeZ7w7aCvJD7ugkw==", + "dev": true, + "requires": { + "is-fullwidth-code-point": "^2.0.0", + "strip-ansi": "^4.0.0" + } + }, + "strip-ansi": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-4.0.0.tgz", + "integrity": "sha512-4XaJ2zQdCzROZDivEVIDPkcQn8LMFSa8kj8Gxb/Lnwzv9A8VctNZ+lfivC/sV3ivW8ElJTERXZoPBRrZKkNKow==", + "dev": true, + "requires": { + "ansi-regex": "^3.0.0" + } + }, + "strip-json-comments": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/strip-json-comments/-/strip-json-comments-2.0.1.tgz", + "integrity": "sha512-4gB8na07fecVVkOI6Rs4e7T6NOTki5EmL7TUduTs6bu3EdnSycntVJ4re8kgZA+wx9IueI2Y11bfbgwtzuE0KQ==", + "dev": true + } + } + }, + "prettier-linter-helpers": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/prettier-linter-helpers/-/prettier-linter-helpers-1.0.0.tgz", + "integrity": "sha512-GbK2cP9nraSSUF9N2XwUwqfzlAFlMNYYl+ShE/V+H8a9uNl/oUqB1w2EL54Jh0OlyRSd8RfWYJ3coVS4TROP2w==", + "dev": true, + "requires": { + "fast-diff": "^1.1.2" + } + }, + "pretty-format": { + "version": "23.6.0", + "resolved": "https://registry.npmjs.org/pretty-format/-/pretty-format-23.6.0.tgz", + "integrity": "sha512-zf9NV1NSlDLDjycnwm6hpFATCGl/K1lt0R/GdkAK2O5LN/rwJoB+Mh93gGJjut4YbmecbfgLWVGSTCr0Ewvvbw==", + "dev": true, + "requires": { + "ansi-regex": "^3.0.0", + "ansi-styles": "^3.2.0" + }, + "dependencies": { + "ansi-regex": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-3.0.0.tgz", + "integrity": "sha512-wFUFA5bg5dviipbQQ32yOQhl6gcJaJXiHE7dvR8VYPG97+J/GNC5FKGepKdEDUFeXRzDxPF1X/Btc8L+v7oqIQ==", + "dev": true + } + } + }, "pretty-ms": { "version": "4.0.0", "resolved": "https://registry.npmjs.org/pretty-ms/-/pretty-ms-4.0.0.tgz", @@ -1785,6 +3885,12 @@ "resolved": "https://registry.npmjs.org/process-nextick-args/-/process-nextick-args-2.0.1.tgz", "integrity": "sha512-3ouUOpQhtgrbOa17J7+uxOTpITYWaGP7/AhoR3+A+/1e9skrzelGi/dXzEYyvbxubEF6Wn2ypscTKiKJFFn1ag==" }, + "progress": { + "version": "2.0.3", + "resolved": "https://registry.npmjs.org/progress/-/progress-2.0.3.tgz", + "integrity": "sha512-7PiHtLll5LdnKIMw100I+8xJXR5gW2QwWYkT6iJva0bXitZKa/XMrSbdmg3r2Xnaidz9Qumd0VPaMrZlF9V9sA==", + "dev": true + }, "prom-client": { "version": "11.5.3", "resolved": "https://registry.npmjs.org/prom-client/-/prom-client-11.5.3.tgz", @@ -1849,6 +3955,18 @@ "resolved": "https://registry.npmjs.org/qs/-/qs-6.7.0.tgz", "integrity": "sha512-VCdBRNFTX1fyE7Nb6FYoURo/SPe62QCaAyzJvUjwRaIsc+NePBEniHlvxFmmX56+HZphIGtV0XeCirBtpDrTyQ==" }, + "quick-lru": { + "version": "4.0.1", + "resolved": "https://registry.npmjs.org/quick-lru/-/quick-lru-4.0.1.tgz", + "integrity": "sha512-ARhCpm70fzdcvNQfPoy49IaanKkTlRWF2JMzqhcJbhSFRZv7nPTvZJdcY7301IPmvW+/p0RgIWnQDLJxifsQ7g==", + "dev": true + }, + "ramda": { + "version": "0.27.0", + "resolved": "https://registry.npmjs.org/ramda/-/ramda-0.27.0.tgz", + "integrity": "sha512-pVzZdDpWwWqEVVLshWUHjNwuVP7SfcmPraYuqocJp1yo2U1R7P+5QAfDhdItkuoGqIBnBYrtPp7rEPqDn9HlZA==", + "dev": true + }, "random-bytes": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/random-bytes/-/random-bytes-1.0.0.tgz", @@ -1882,6 +4000,27 @@ "unpipe": "1.0.0" } }, + "read-pkg": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/read-pkg/-/read-pkg-2.0.0.tgz", + "integrity": "sha512-eFIBOPW7FGjzBuk3hdXEuNSiTZS/xEMlH49HxMyzb0hyPfu4EhVjT2DH32K1hSSmVq4sebAWnZuuY5auISUTGA==", + "dev": true, + "requires": { + "load-json-file": "^2.0.0", + "normalize-package-data": "^2.3.2", + "path-type": "^2.0.0" + } + }, + "read-pkg-up": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/read-pkg-up/-/read-pkg-up-2.0.0.tgz", + "integrity": "sha512-1orxQfbWGUiTn9XsPlChs6rLie/AV9jwZTGmu2NZw/CUDJQchXJFYE0Fq5j7+n558T1JhDWLdhyd1Zj+wLY//w==", + "dev": true, + "requires": { + "find-up": "^2.0.0", + "read-pkg": "^2.0.0" + } + }, "readable-stream": { "version": "2.3.7", "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-2.3.7.tgz", @@ -1963,13 +4102,48 @@ "mkdirp": "~0.3.5" } }, + "debug": { + "version": "4.1.1", + "resolved": "https://registry.npmjs.org/debug/-/debug-4.1.1.tgz", + "integrity": "sha512-pYAIzeRo8J6KPEaJ0VWOh5Pzkbw/RetuzehGM7QRRX5he4fPHx2rdKMB256ehJCkX+XRQm16eZLqLNS8RSZXZw==", + "requires": { + "ms": "^2.1.1" + } + }, + "ioredis": { + "version": "4.16.3", + "resolved": "https://registry.npmjs.org/ioredis/-/ioredis-4.16.3.tgz", + "integrity": "sha512-Ejvcs2yW19Vq8AipvbtfcX3Ig8XG9EAyFOvGbhI/Q1QoVOK9ZdgY092kdOyOWIYBnPHjfjMJhU9qhsnp0i0K1w==", + "requires": { + "cluster-key-slot": "^1.1.0", + "debug": "^4.1.1", + "denque": "^1.1.0", + "lodash.defaults": "^4.2.0", + "lodash.flatten": "^4.4.0", + "redis-commands": "1.5.0", + "redis-errors": "^1.2.0", + "redis-parser": "^3.0.0", + "standard-as-callback": "^2.0.1" + } + }, "mkdirp": { "version": "0.3.5", "resolved": "https://registry.npmjs.org/mkdirp/-/mkdirp-0.3.5.tgz", "integrity": "sha512-8OCq0De/h9ZxseqzCH8Kw/Filf5pF/vMI6+BH7Lu0jXz2pqYCjTAQRolSxRIi+Ax+oCCjlxoJMP0YQ4XlrQNHg==" + }, + "ms": { + "version": "2.1.2", + "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.2.tgz", + "integrity": "sha512-sGkPx+VjMtmA6MX27oA4FBFELFCZZ4S4XqeGOXCv68tT+jb3vk/RyaKWP0PTKyWtmLSM0b+adUTEvbs1PEaH2w==" } } }, + "regexpp": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/regexpp/-/regexpp-2.0.1.tgz", + "integrity": "sha512-lv0M6+TkDVniA3aD1Eg0DVpfU/booSu7Eev3TDO/mZKHBfVjgCGTV4t4buppESEYDtkArYFOxTJWv6S5C+iaNw==", + "dev": true + }, "request": { "version": "2.88.2", "resolved": "https://registry.npmjs.org/request/-/request-2.88.2.tgz", @@ -2036,6 +4210,12 @@ } } }, + "require-directory": { + "version": "2.1.1", + "resolved": "https://registry.npmjs.org/require-directory/-/require-directory-2.1.1.tgz", + "integrity": "sha512-fGxEI7+wsG9xrvdjsrlmL22OMTTiHRwAMroiEeMgq8gzoLC/PQr7RsRDSTLUg/bZAZtF+TVIkHc6/4RIKrui+Q==", + "dev": true + }, "require-in-the-middle": { "version": "4.0.1", "resolved": "https://registry.npmjs.org/require-in-the-middle/-/require-in-the-middle-4.0.1.tgz", @@ -2067,6 +4247,18 @@ "integrity": "sha512-oyrU88skkMtDdauHDuKVrgR+zuItqr6/c//FXzvmxRGMexSDc6hNvJInGW3LL46n+8b50RykrvwSUIIQH2LQ5A==", "dev": true }, + "require-main-filename": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/require-main-filename/-/require-main-filename-2.0.0.tgz", + "integrity": "sha512-NKN5kMDylKuldxYLSUfrbo5Tuzh4hd+2E8NPPX02mZtn1VuREQToYe/ZdlJy+J3uCpfaiGF05e7B8W0iXbQHmg==", + "dev": true + }, + "require-relative": { + "version": "0.8.7", + "resolved": "https://registry.npmjs.org/require-relative/-/require-relative-0.8.7.tgz", + "integrity": "sha512-AKGr4qvHiryxRb19m3PsLRGuKVAbJLUD7E6eOaHkfKhwc+vSgVOCY5xNvm9EkolBKTOf0GrQAZKLimOCz81Khg==", + "dev": true + }, "resolve": { "version": "1.15.1", "resolved": "https://registry.npmjs.org/resolve/-/resolve-1.15.1.tgz", @@ -2075,6 +4267,22 @@ "path-parse": "^1.0.6" } }, + "resolve-from": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/resolve-from/-/resolve-from-4.0.0.tgz", + "integrity": "sha512-pb/MYmXstAkysRFx8piNI1tGFNQIFA3vkE3Gq4EuA1dF6gHp/+vgZqsCGJapvy8N3Q+4o7FwvquPJcnZ7RYy4g==", + "dev": true + }, + "restore-cursor": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/restore-cursor/-/restore-cursor-3.1.0.tgz", + "integrity": "sha512-l+sSefzHpj5qimhFSE5a8nufZYAM3sBSVMAPtYkmC+4EH2anSGaEMXSD0izRQbu9nfyQ9y5JrVmp7E8oZrUjvA==", + "dev": true, + "requires": { + "onetime": "^5.1.0", + "signal-exit": "^3.0.2" + } + }, "retry-axios": { "version": "0.3.2", "resolved": "https://registry.npmjs.org/retry-axios/-/retry-axios-0.3.2.tgz", @@ -2113,6 +4321,21 @@ "glob": "^6.0.1" } }, + "run-async": { + "version": "2.4.1", + "resolved": "https://registry.npmjs.org/run-async/-/run-async-2.4.1.tgz", + "integrity": "sha512-tvVnVv01b8c1RrA6Ep7JkStj85Guv/YrMcwqYQnwjsAS2cTmmPGBBjAjpCW7RrSodNSoE2/qg9O4bceNvUuDgQ==", + "dev": true + }, + "rxjs": { + "version": "6.5.5", + "resolved": "https://registry.npmjs.org/rxjs/-/rxjs-6.5.5.tgz", + "integrity": "sha512-WfQI+1gohdf0Dai/Bbmk5L5ItH5tYqm3ki2c5GdWhKjalzjg93N3avFjVStyZZz+A2Em+ZxKH5bNghw9UeylGQ==", + "dev": true, + "requires": { + "tslib": "^1.9.0" + } + }, "safe-buffer": { "version": "5.1.1", "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.1.1.tgz", @@ -2196,6 +4419,12 @@ "send": "0.17.1" } }, + "set-blocking": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/set-blocking/-/set-blocking-2.0.0.tgz", + "integrity": "sha512-KiKBS8AnWGEyLzofFfmvKwpdPzqiy16LvQfK3yv/fVH7Bj13/wl3JSR1J+rfgRE9q7xUJK4qvgS8raSOeLUehw==", + "dev": true + }, "setprototypeof": { "version": "1.1.1", "resolved": "https://registry.npmjs.org/setprototypeof/-/setprototypeof-1.1.1.tgz", @@ -2216,11 +4445,32 @@ } } }, + "shebang-command": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/shebang-command/-/shebang-command-1.2.0.tgz", + "integrity": "sha512-EV3L1+UQWGor21OmnvojK36mhg+TyIKDh3iFBKBohr5xeXIhNBcx8oWdgkTEEQ+BEFFYdLRuqMfd5L84N1V5Vg==", + "dev": true, + "requires": { + "shebang-regex": "^1.0.0" + } + }, + "shebang-regex": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/shebang-regex/-/shebang-regex-1.0.0.tgz", + "integrity": "sha512-wpoSFAxys6b2a2wHZ1XpDSgD7N9iVjg29Ph9uV/uaP9Ex/KXlkTZTeddxDPSYQpgvzKLGJke2UU0AzoGCjNIvQ==", + "dev": true + }, "shimmer": { "version": "1.2.1", "resolved": "https://registry.npmjs.org/shimmer/-/shimmer-1.2.1.tgz", "integrity": "sha512-sQTKC1Re/rM6XyFM6fIAGHRPVGvyXfgzIDvzoq608vM+jeyVD0Tu1E6Np0Kc2zAIFWIj963V2800iF/9LPieQw==" }, + "signal-exit": { + "version": "3.0.3", + "resolved": "https://registry.npmjs.org/signal-exit/-/signal-exit-3.0.3.tgz", + "integrity": "sha512-VUJ49FC8U1OxwZLxIbTTrDvLnf/6TDgxZcK8wxR8zs13xpx7xbG60ndBlhNrFi2EMuFRoeDoJO7wthSLq42EjA==", + "dev": true + }, "sinon": { "version": "2.4.1", "resolved": "https://registry.npmjs.org/sinon/-/sinon-2.4.1.tgz", @@ -2260,6 +4510,25 @@ } } }, + "slice-ansi": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/slice-ansi/-/slice-ansi-2.1.0.tgz", + "integrity": "sha512-Qu+VC3EwYLldKa1fCxuuvULvSJOKEgk9pi8dZeCVK7TqBfUNTH4sFkk4joj8afVSfAYgJoSOetjx9QWOJ5mYoQ==", + "dev": true, + "requires": { + "ansi-styles": "^3.2.0", + "astral-regex": "^1.0.0", + "is-fullwidth-code-point": "^2.0.0" + }, + "dependencies": { + "is-fullwidth-code-point": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/is-fullwidth-code-point/-/is-fullwidth-code-point-2.0.0.tgz", + "integrity": "sha512-VHskAKYM8RfSFXwee5t5cbN5PZeq1Wrh6qd5bkyiXIf6UQcN6w/A0eXM9r6t8d+GYOh+o6ZhiEnb88LN/Y8m2w==", + "dev": true + } + } + }, "socket.io": { "version": "0.9.19", "resolved": "https://registry.npmjs.org/socket.io/-/socket.io-0.9.19.tgz", @@ -2306,6 +4575,38 @@ "resolved": "https://registry.npmjs.org/source-map/-/source-map-0.6.1.tgz", "integrity": "sha512-UjgapumWlbMhkBgzT7Ykc5YXUT46F0iKu8SGXq0bcwP5dz/h0Plj6enJqjz1Zbq2l5WaqYnrVbwWOWMyF3F47g==" }, + "spdx-correct": { + "version": "3.1.1", + "resolved": "https://registry.npmjs.org/spdx-correct/-/spdx-correct-3.1.1.tgz", + "integrity": "sha512-cOYcUWwhCuHCXi49RhFRCyJEK3iPj1Ziz9DpViV3tbZOwXD49QzIN3MpOLJNxh2qwq2lJJZaKMVw9qNi4jTC0w==", + "dev": true, + "requires": { + "spdx-expression-parse": "^3.0.0", + "spdx-license-ids": "^3.0.0" + } + }, + "spdx-exceptions": { + "version": "2.3.0", + "resolved": "https://registry.npmjs.org/spdx-exceptions/-/spdx-exceptions-2.3.0.tgz", + "integrity": "sha512-/tTrYOC7PPI1nUAgx34hUpqXuyJG+DTHJTnIULG4rDygi4xu/tfgmq1e1cIRwRzwZgo4NLySi+ricLkZkw4i5A==", + "dev": true + }, + "spdx-expression-parse": { + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/spdx-expression-parse/-/spdx-expression-parse-3.0.1.tgz", + "integrity": "sha512-cbqHunsQWnJNE6KhVSMsMeH5H/L9EpymbzqTQ3uLwNCLZ1Q481oWaofqH7nO6V07xlXwY6PhQdQ2IedWx/ZK4Q==", + "dev": true, + "requires": { + "spdx-exceptions": "^2.1.0", + "spdx-license-ids": "^3.0.0" + } + }, + "spdx-license-ids": { + "version": "3.0.5", + "resolved": "https://registry.npmjs.org/spdx-license-ids/-/spdx-license-ids-3.0.5.tgz", + "integrity": "sha512-J+FWzZoynJEXGphVIS+XEh3kFSjZX/1i9gFBaWQcB+/tmpe2qUsSBABpcxqxnAxFdiUFEgAX1bjYGQvIZmoz9Q==", + "dev": true + }, "split": { "version": "1.0.1", "resolved": "https://registry.npmjs.org/split/-/split-1.0.1.tgz", @@ -2314,6 +4615,12 @@ "through": "2" } }, + "sprintf-js": { + "version": "1.0.3", + "resolved": "https://registry.npmjs.org/sprintf-js/-/sprintf-js-1.0.3.tgz", + "integrity": "sha512-D9cPgkvLlV3t3IzL0D0YLvGA9Ahk4PcvVwUbN0dSGr1aP0Nrt4AEnTUbuGvquEC0mA64Gqt1fzirlRs5ibXx8g==", + "dev": true + }, "sshpk": { "version": "1.16.1", "resolved": "https://registry.npmjs.org/sshpk/-/sshpk-1.16.1.tgz", @@ -2355,6 +4662,48 @@ "resolved": "https://registry.npmjs.org/stream-shift/-/stream-shift-1.0.1.tgz", "integrity": "sha512-AiisoFqQ0vbGcZgQPY1cdP2I76glaVA/RauYR4G4thNFgkTqr90yXTo4LYX60Jl+sIlPNHHdGSwo01AvbKUSVQ==" }, + "string-width": { + "version": "4.2.0", + "resolved": "https://registry.npmjs.org/string-width/-/string-width-4.2.0.tgz", + "integrity": "sha512-zUz5JD+tgqtuDjMhwIg5uFVV3dtqZ9yQJlZVfq4I01/K5Paj5UHj7VyrQOJvzawSVlKpObApbfD0Ed6yJc+1eg==", + "dev": true, + "requires": { + "emoji-regex": "^8.0.0", + "is-fullwidth-code-point": "^3.0.0", + "strip-ansi": "^6.0.0" + }, + "dependencies": { + "strip-ansi": { + "version": "6.0.0", + "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-6.0.0.tgz", + "integrity": "sha512-AuvKTrTfQNYNIctbR1K/YGTR1756GycPsg7b9bdV9Duqur4gv6aKqHXah67Z8ImS7WEz5QVcOtlfW2rZEugt6w==", + "dev": true, + "requires": { + "ansi-regex": "^5.0.0" + } + } + } + }, + "string.prototype.trimend": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/string.prototype.trimend/-/string.prototype.trimend-1.0.1.tgz", + "integrity": "sha512-LRPxFUaTtpqYsTeNKaFOw3R4bxIzWOnbQ837QfBylo8jIxtcbK/A/sMV7Q+OAV/vWo+7s25pOE10KYSjaSO06g==", + "dev": true, + "requires": { + "define-properties": "^1.1.3", + "es-abstract": "^1.17.5" + } + }, + "string.prototype.trimstart": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/string.prototype.trimstart/-/string.prototype.trimstart-1.0.1.tgz", + "integrity": "sha512-XxZn+QpvrBI1FOcg6dIpxUPgWCPuNXvMD72aaRaUQv1eD4e/Qy8i/hFTe0BUmD60p/QA6bh1avmuPTfNjqVWRw==", + "dev": true, + "requires": { + "define-properties": "^1.1.3", + "es-abstract": "^1.17.5" + } + }, "string_decoder": { "version": "1.1.1", "resolved": "https://registry.npmjs.org/string_decoder/-/string_decoder-1.1.1.tgz", @@ -2363,6 +4712,81 @@ "safe-buffer": "~5.1.0" } }, + "strip-ansi": { + "version": "5.2.0", + "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-5.2.0.tgz", + "integrity": "sha512-DuRs1gKbBqsMKIZlrffwlug8MHkcnpjs5VPmL1PAh+mA30U0DTotfDZ0d2UUsXpPmPmMMJ6W773MaA3J+lbiWA==", + "dev": true, + "requires": { + "ansi-regex": "^4.1.0" + }, + "dependencies": { + "ansi-regex": { + "version": "4.1.0", + "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-4.1.0.tgz", + "integrity": "sha512-1apePfXM1UOSqw0o9IiFAovVz9M5S1Dg+4TrDwfMewQ6p/rmMueb7tWZjQ1rx4Loy1ArBggoqGpfqqdI4rondg==", + "dev": true + } + } + }, + "strip-bom": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/strip-bom/-/strip-bom-3.0.0.tgz", + "integrity": "sha512-vavAMRXOgBVNF6nyEEmL3DBK19iRpDcoIwW+swQ+CbGiu7lju6t+JklA1MHweoWtadgt4ISVUsXLyDq34ddcwA==", + "dev": true + }, + "strip-json-comments": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/strip-json-comments/-/strip-json-comments-3.1.0.tgz", + "integrity": "sha512-e6/d0eBu7gHtdCqFt0xJr642LdToM5/cN4Qb9DbHjVx1CP5RyeM+zH7pbecEmDv/lBqb0QH+6Uqq75rxFPkM0w==", + "dev": true + }, + "supports-color": { + "version": "5.5.0", + "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-5.5.0.tgz", + "integrity": "sha512-QjVjwdXIt408MIiAqCX4oUKsgU2EqAGzs2Ppkm4aQYbjm+ZEWEcW4SfFNTr4uMNZma0ey4f5lgLrkB0aX0QMow==", + "dev": true, + "requires": { + "has-flag": "^3.0.0" + } + }, + "table": { + "version": "5.4.6", + "resolved": "https://registry.npmjs.org/table/-/table-5.4.6.tgz", + "integrity": "sha512-wmEc8m4fjnob4gt5riFRtTu/6+4rSe12TpAELNSqHMfF3IqnA+CH37USM6/YR3qRZv7e56kAEAtd6nKZaxe0Ug==", + "dev": true, + "requires": { + "ajv": "^6.10.2", + "lodash": "^4.17.14", + "slice-ansi": "^2.1.0", + "string-width": "^3.0.0" + }, + "dependencies": { + "emoji-regex": { + "version": "7.0.3", + "resolved": "https://registry.npmjs.org/emoji-regex/-/emoji-regex-7.0.3.tgz", + "integrity": "sha512-CwBLREIQ7LvYFB0WyRvwhq5N5qPhc6PMjD6bYggFlI5YyDgl+0vxq5VHbMOFqLg7hfWzmu8T5Z1QofhmTIhItA==", + "dev": true + }, + "is-fullwidth-code-point": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/is-fullwidth-code-point/-/is-fullwidth-code-point-2.0.0.tgz", + "integrity": "sha512-VHskAKYM8RfSFXwee5t5cbN5PZeq1Wrh6qd5bkyiXIf6UQcN6w/A0eXM9r6t8d+GYOh+o6ZhiEnb88LN/Y8m2w==", + "dev": true + }, + "string-width": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/string-width/-/string-width-3.1.0.tgz", + "integrity": "sha512-vafcv6KjVZKSgz06oM/H6GDBrAtz8vdhQakGjFIvNrHA6y3HCF1CInLy+QLq8dTJPQ1b+KDUqDFctkdRW44e1w==", + "dev": true, + "requires": { + "emoji-regex": "^7.0.1", + "is-fullwidth-code-point": "^2.0.0", + "strip-ansi": "^5.1.0" + } + } + } + }, "tdigest": { "version": "0.1.1", "resolved": "https://registry.npmjs.org/tdigest/-/tdigest-0.1.1.tgz", @@ -2394,6 +4818,12 @@ "integrity": "sha512-hJnc6Qg3dWoOMkqP53F0dzRIgtmsAge09kxUIqGrEUS4qr5rWLckGYaQAVr+opBrIMRErGgy6f5aPnyPpyGRfg==", "dev": true }, + "text-table": { + "version": "0.2.0", + "resolved": "https://registry.npmjs.org/text-table/-/text-table-0.2.0.tgz", + "integrity": "sha512-N+8UisAXDGk8PFXP4HAzVR9nbfmVJ3zYLAWiTIoqC5v5isinhr+r5uaO8+7r3BMfuNIufIsA7RdpVgacC2cSpw==", + "dev": true + }, "through": { "version": "2.3.8", "resolved": "https://registry.npmjs.org/through/-/through-2.3.8.tgz", @@ -2418,6 +4848,15 @@ "resolved": "https://registry.npmjs.org/tinycolor/-/tinycolor-0.0.1.tgz", "integrity": "sha512-+CorETse1kl98xg0WAzii8DTT4ABF4R3nquhrkIbVGcw1T8JYs5Gfx9xEfGINPUZGDj9C4BmOtuKeaTtuuRolg==" }, + "tmp": { + "version": "0.0.33", + "resolved": "https://registry.npmjs.org/tmp/-/tmp-0.0.33.tgz", + "integrity": "sha512-jRCJlojKnZ3addtTOjdIqoRuPEKBvNXcGYqzO6zWZX8KfKEpnGY5jfggJQ3EjKuu8D4bJRr0y+cYJFmYbImXGw==", + "dev": true, + "requires": { + "os-tmpdir": "~1.0.2" + } + }, "toidentifier": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/toidentifier/-/toidentifier-1.0.0.tgz", @@ -2439,6 +4878,32 @@ } } }, + "tsconfig-paths": { + "version": "3.9.0", + "resolved": "https://registry.npmjs.org/tsconfig-paths/-/tsconfig-paths-3.9.0.tgz", + "integrity": "sha512-dRcuzokWhajtZWkQsDVKbWyY+jgcLC5sqJhg2PSgf4ZkH2aHPvaOY8YWGhmjb68b5qqTfasSsDO9k7RUiEmZAw==", + "dev": true, + "requires": { + "@types/json5": "^0.0.29", + "json5": "^1.0.1", + "minimist": "^1.2.0", + "strip-bom": "^3.0.0" + }, + "dependencies": { + "minimist": { + "version": "1.2.5", + "resolved": "https://registry.npmjs.org/minimist/-/minimist-1.2.5.tgz", + "integrity": "sha512-FM9nNUYrRBAELZQT3xeZQ7fmMOBg6nWNmJKTcgsJeaLstP/UODVpGsr5OhXhhXg6f+qtJ8uiZ+PUxkDWcgIXLw==", + "dev": true + } + } + }, + "tslib": { + "version": "1.13.0", + "resolved": "https://registry.npmjs.org/tslib/-/tslib-1.13.0.tgz", + "integrity": "sha512-i/6DQjL8Xf3be4K/E6Wgpekn5Qasl1usyw++dAA35Ue5orEn65VIxOA+YvNNl9HV3qv70T7CNwjODHZrLwvd1Q==", + "dev": true + }, "tunnel-agent": { "version": "0.6.0", "resolved": "https://registry.npmjs.org/tunnel-agent/-/tunnel-agent-0.6.0.tgz", @@ -2452,12 +4917,27 @@ "resolved": "https://registry.npmjs.org/tweetnacl/-/tweetnacl-0.14.5.tgz", "integrity": "sha512-KXXFFdAbFXY4geFIwoyNK+f5Z1b7swfXABfL7HXCmoIWMKU3dmS26672A4EeQtDzLKy7SXmfBu51JolvEKwtGA==" }, + "type-check": { + "version": "0.3.2", + "resolved": "https://registry.npmjs.org/type-check/-/type-check-0.3.2.tgz", + "integrity": "sha512-ZCmOJdvOWDBYJlzAoFkC+Q0+bUyEOS1ltgp1MGU03fqHG+dbi9tBFU2Rd9QKiDZFAYrhPh2JUf7rZRIuHRKtOg==", + "dev": true, + "requires": { + "prelude-ls": "~1.1.2" + } + }, "type-detect": { "version": "0.1.1", "resolved": "https://registry.npmjs.org/type-detect/-/type-detect-0.1.1.tgz", "integrity": "sha512-5rqszGVwYgBoDkIm2oUtvkfZMQ0vk29iDMU0W2qCa3rG0vPDNczCMT4hV/bLBgLg8k8ri6+u3Zbt+S/14eMzlA==", "dev": true }, + "type-fest": { + "version": "0.8.1", + "resolved": "https://registry.npmjs.org/type-fest/-/type-fest-0.8.1.tgz", + "integrity": "sha512-4dbzIzqvjtgiM5rw1k5rEHtBANKmdudhGyBEajN01fEyhaAIhsoKNy6y7+IN93IfpFtwY9iqi7kD+xwKhQsNJA==", + "dev": true + }, "type-is": { "version": "1.6.18", "resolved": "https://registry.npmjs.org/type-is/-/type-is-1.6.18.tgz", @@ -2482,6 +4962,12 @@ } } }, + "typescript": { + "version": "3.9.5", + "resolved": "https://registry.npmjs.org/typescript/-/typescript-3.9.5.tgz", + "integrity": "sha512-hSAifV3k+i6lEoCJ2k6R2Z/rp/H3+8sdmcn5NrS3/3kE7+RyZXm9aqvxWqjEXHAd8b0pShatpcdMTvEdvAJltQ==", + "dev": true + }, "uglify-js": { "version": "1.2.5", "resolved": "https://registry.npmjs.org/uglify-js/-/uglify-js-1.2.5.tgz", @@ -2528,6 +5014,22 @@ "resolved": "https://registry.npmjs.org/uuid/-/uuid-3.0.0.tgz", "integrity": "sha512-rqE1LoOVLv3QrZMjb4NkF5UWlkurCfPyItVnFPNKDDGkHw4dQUdE4zMcLqx28+0Kcf3+bnUk4PisaiRJT4aiaQ==" }, + "v8-compile-cache": { + "version": "2.1.1", + "resolved": "https://registry.npmjs.org/v8-compile-cache/-/v8-compile-cache-2.1.1.tgz", + "integrity": "sha512-8OQ9CL+VWyt3JStj7HX7/ciTL2V3Rl1Wf5OL+SNTm0yK1KvtReVulksyeRnCANHHuUxHlQig+JJDlUhBt1NQDQ==", + "dev": true + }, + "validate-npm-package-license": { + "version": "3.0.4", + "resolved": "https://registry.npmjs.org/validate-npm-package-license/-/validate-npm-package-license-3.0.4.tgz", + "integrity": "sha512-DpKm2Ui/xN7/HQKCtpZxoRWBhZ9Z0kqtygG8XCgNQ8ZlDnxuQmWhj566j8fN4Cu3/JmbhsDo7fcAJq4s9h27Ew==", + "dev": true, + "requires": { + "spdx-correct": "^3.0.0", + "spdx-expression-parse": "^3.0.0" + } + }, "vary": { "version": "1.1.2", "resolved": "https://registry.npmjs.org/vary/-/vary-1.1.2.tgz", @@ -2543,11 +5045,151 @@ "extsprintf": "^1.2.0" } }, + "vue-eslint-parser": { + "version": "2.0.3", + "resolved": "https://registry.npmjs.org/vue-eslint-parser/-/vue-eslint-parser-2.0.3.tgz", + "integrity": "sha512-ZezcU71Owm84xVF6gfurBQUGg8WQ+WZGxgDEQu1IHFBZNx7BFZg3L1yHxrCBNNwbwFtE1GuvfJKMtb6Xuwc/Bw==", + "dev": true, + "requires": { + "debug": "^3.1.0", + "eslint-scope": "^3.7.1", + "eslint-visitor-keys": "^1.0.0", + "espree": "^3.5.2", + "esquery": "^1.0.0", + "lodash": "^4.17.4" + }, + "dependencies": { + "acorn": { + "version": "5.7.4", + "resolved": "https://registry.npmjs.org/acorn/-/acorn-5.7.4.tgz", + "integrity": "sha512-1D++VG7BhrtvQpNbBzovKNc1FLGGEE/oGe7b9xJm/RFHMBeUaUGpluV9RLjZa47YFdPcDAenEYuq9pQPcMdLJg==", + "dev": true + }, + "acorn-jsx": { + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/acorn-jsx/-/acorn-jsx-3.0.1.tgz", + "integrity": "sha512-AU7pnZkguthwBjKgCg6998ByQNIMjbuDQZ8bb78QAFZwPfmKia8AIzgY/gWgqCjnht8JLdXmB4YxA0KaV60ncQ==", + "dev": true, + "requires": { + "acorn": "^3.0.4" + }, + "dependencies": { + "acorn": { + "version": "3.3.0", + "resolved": "https://registry.npmjs.org/acorn/-/acorn-3.3.0.tgz", + "integrity": "sha512-OLUyIIZ7mF5oaAUT1w0TFqQS81q3saT46x8t7ukpPjMNk+nbs4ZHhs7ToV8EWnLYLepjETXd4XaCE4uxkMeqUw==", + "dev": true + } + } + }, + "debug": { + "version": "3.2.6", + "resolved": "https://registry.npmjs.org/debug/-/debug-3.2.6.tgz", + "integrity": "sha512-mel+jf7nrtEl5Pn1Qx46zARXKDpBbvzezse7p7LqINmdoIk8PYP5SySaxEmYv6TZ0JyEKA1hsCId6DIhgITtWQ==", + "dev": true, + "requires": { + "ms": "^2.1.1" + } + }, + "eslint-scope": { + "version": "3.7.3", + "resolved": "https://registry.npmjs.org/eslint-scope/-/eslint-scope-3.7.3.tgz", + "integrity": "sha512-W+B0SvF4gamyCTmUc+uITPY0989iXVfKvhwtmJocTaYoc/3khEHmEmvfY/Gn9HA9VV75jrQECsHizkNw1b68FA==", + "dev": true, + "requires": { + "esrecurse": "^4.1.0", + "estraverse": "^4.1.1" + } + }, + "espree": { + "version": "3.5.4", + "resolved": "https://registry.npmjs.org/espree/-/espree-3.5.4.tgz", + "integrity": "sha512-yAcIQxtmMiB/jL32dzEp2enBeidsB7xWPLNiw3IIkpVds1P+h7qF9YwJq1yUNzp2OKXgAprs4F61ih66UsoD1A==", + "dev": true, + "requires": { + "acorn": "^5.5.0", + "acorn-jsx": "^3.0.0" + } + }, + "ms": { + "version": "2.1.2", + "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.2.tgz", + "integrity": "sha512-sGkPx+VjMtmA6MX27oA4FBFELFCZZ4S4XqeGOXCv68tT+jb3vk/RyaKWP0PTKyWtmLSM0b+adUTEvbs1PEaH2w==", + "dev": true + } + } + }, + "which": { + "version": "1.3.1", + "resolved": "https://registry.npmjs.org/which/-/which-1.3.1.tgz", + "integrity": "sha512-HxJdYWq1MTIQbJ3nw0cqssHoTNU267KlrDuGZ1WYlxDStUtKUhOaJmh112/TZmHxxUfuJqPXSOm7tDyas0OSIQ==", + "dev": true, + "requires": { + "isexe": "^2.0.0" + } + }, + "which-module": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/which-module/-/which-module-2.0.0.tgz", + "integrity": "sha512-B+enWhmw6cjfVC7kS8Pj9pCrKSc5txArRyaYGe088shv/FGWH+0Rjx/xPgtsWfsUtS27FkP697E4DDhgrgoc0Q==", + "dev": true + }, + "word-wrap": { + "version": "1.2.3", + "resolved": "https://registry.npmjs.org/word-wrap/-/word-wrap-1.2.3.tgz", + "integrity": "sha512-Hz/mrNwitNRh/HUAtM/VT/5VH+ygD6DV7mYKZAtHOrbs8U7lvPS6xf7EJKMF0uW1KJCl0H701g3ZGus+muE5vQ==", + "dev": true + }, + "wrap-ansi": { + "version": "5.1.0", + "resolved": "https://registry.npmjs.org/wrap-ansi/-/wrap-ansi-5.1.0.tgz", + "integrity": "sha512-QC1/iN/2/RPVJ5jYK8BGttj5z83LmSKmvbvrXPNCLZSEb32KKVDJDl/MOt2N01qU2H/FkzEa9PKto1BqDjtd7Q==", + "dev": true, + "requires": { + "ansi-styles": "^3.2.0", + "string-width": "^3.0.0", + "strip-ansi": "^5.0.0" + }, + "dependencies": { + "emoji-regex": { + "version": "7.0.3", + "resolved": "https://registry.npmjs.org/emoji-regex/-/emoji-regex-7.0.3.tgz", + "integrity": "sha512-CwBLREIQ7LvYFB0WyRvwhq5N5qPhc6PMjD6bYggFlI5YyDgl+0vxq5VHbMOFqLg7hfWzmu8T5Z1QofhmTIhItA==", + "dev": true + }, + "is-fullwidth-code-point": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/is-fullwidth-code-point/-/is-fullwidth-code-point-2.0.0.tgz", + "integrity": "sha512-VHskAKYM8RfSFXwee5t5cbN5PZeq1Wrh6qd5bkyiXIf6UQcN6w/A0eXM9r6t8d+GYOh+o6ZhiEnb88LN/Y8m2w==", + "dev": true + }, + "string-width": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/string-width/-/string-width-3.1.0.tgz", + "integrity": "sha512-vafcv6KjVZKSgz06oM/H6GDBrAtz8vdhQakGjFIvNrHA6y3HCF1CInLy+QLq8dTJPQ1b+KDUqDFctkdRW44e1w==", + "dev": true, + "requires": { + "emoji-regex": "^7.0.1", + "is-fullwidth-code-point": "^2.0.0", + "strip-ansi": "^5.1.0" + } + } + } + }, "wrappy": { "version": "1.0.2", "resolved": "https://registry.npmjs.org/wrappy/-/wrappy-1.0.2.tgz", "integrity": "sha512-l4Sp/DRseor9wL6EvV2+TuQn63dMkPjZ/sp9XkghTEbV9KlPS1xUsZ3u7/IQO4wxtcFB4bgpQPRcR3QCvezPcQ==" }, + "write": { + "version": "1.0.3", + "resolved": "https://registry.npmjs.org/write/-/write-1.0.3.tgz", + "integrity": "sha512-/lg70HAjtkUgWPVZhZcm+T4hkL8Zbtp1nFNOn3lRrxnlv50SRBv7cR7RqR+GMsd3hUXy9hWBo4CHTbFTcOYwig==", + "dev": true, + "requires": { + "mkdirp": "^0.5.1" + } + }, "ws": { "version": "0.4.32", "resolved": "https://registry.npmjs.org/ws/-/ws-0.4.32.tgz", @@ -2576,11 +5218,98 @@ "resolved": "https://registry.npmjs.org/xmlhttprequest/-/xmlhttprequest-1.4.2.tgz", "integrity": "sha512-WTsthd44hTdCRrHkdtTgbgTKIJyNDV+xiShdooFZBUstY7xk+EXMx/u5gjuUXaCiCWvtBVCHwauzml2joevB4w==" }, + "y18n": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/y18n/-/y18n-4.0.0.tgz", + "integrity": "sha512-r9S/ZyXu/Xu9q1tYlpsLIsa3EeLXXk0VwlxqTcFRfg9EhMW+17kbt9G0NrgCmhGb5vT2hyhJZLfDGx+7+5Uj/w==", + "dev": true + }, "yallist": { "version": "3.1.1", "resolved": "https://registry.npmjs.org/yallist/-/yallist-3.1.1.tgz", "integrity": "sha512-a4UGQaWPH59mOXUYnAG2ewncQS4i4F43Tv3JoAM+s2VDAmS9NsK8GpDMLrCHPksFT7h3K6TOoUNn2pb7RoXx4g==" }, + "yargs": { + "version": "13.3.2", + "resolved": "https://registry.npmjs.org/yargs/-/yargs-13.3.2.tgz", + "integrity": "sha512-AX3Zw5iPruN5ie6xGRIDgqkT+ZhnRlZMLMHAs8tg7nRruy2Nb+i5o9bwghAogtM08q1dpr2LVoS8KSTMYpWXUw==", + "dev": true, + "requires": { + "cliui": "^5.0.0", + "find-up": "^3.0.0", + "get-caller-file": "^2.0.1", + "require-directory": "^2.1.1", + "require-main-filename": "^2.0.0", + "set-blocking": "^2.0.0", + "string-width": "^3.0.0", + "which-module": "^2.0.0", + "y18n": "^4.0.0", + "yargs-parser": "^13.1.2" + }, + "dependencies": { + "emoji-regex": { + "version": "7.0.3", + "resolved": "https://registry.npmjs.org/emoji-regex/-/emoji-regex-7.0.3.tgz", + "integrity": "sha512-CwBLREIQ7LvYFB0WyRvwhq5N5qPhc6PMjD6bYggFlI5YyDgl+0vxq5VHbMOFqLg7hfWzmu8T5Z1QofhmTIhItA==", + "dev": true + }, + "find-up": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/find-up/-/find-up-3.0.0.tgz", + "integrity": "sha512-1yD6RmLI1XBfxugvORwlck6f75tYL+iR0jqwsOrOxMZyGYqUuDhJ0l4AXdO1iX/FTs9cBAMEk1gWSEx1kSbylg==", + "dev": true, + "requires": { + "locate-path": "^3.0.0" + } + }, + "is-fullwidth-code-point": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/is-fullwidth-code-point/-/is-fullwidth-code-point-2.0.0.tgz", + "integrity": "sha512-VHskAKYM8RfSFXwee5t5cbN5PZeq1Wrh6qd5bkyiXIf6UQcN6w/A0eXM9r6t8d+GYOh+o6ZhiEnb88LN/Y8m2w==", + "dev": true + }, + "locate-path": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/locate-path/-/locate-path-3.0.0.tgz", + "integrity": "sha512-7AO748wWnIhNqAuaty2ZWHkQHRSNfPVIsPIfwEOWO22AmaoVrWavlOcMR5nzTLNYvp36X220/maaRsrec1G65A==", + "dev": true, + "requires": { + "p-locate": "^3.0.0", + "path-exists": "^3.0.0" + } + }, + "p-locate": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/p-locate/-/p-locate-3.0.0.tgz", + "integrity": "sha512-x+12w/To+4GFfgJhBEpiDcLozRJGegY+Ei7/z0tSLkMmxGZNybVMSfWj9aJn8Z5Fc7dBUNJOOVgPv2H7IwulSQ==", + "dev": true, + "requires": { + "p-limit": "^2.0.0" + } + }, + "string-width": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/string-width/-/string-width-3.1.0.tgz", + "integrity": "sha512-vafcv6KjVZKSgz06oM/H6GDBrAtz8vdhQakGjFIvNrHA6y3HCF1CInLy+QLq8dTJPQ1b+KDUqDFctkdRW44e1w==", + "dev": true, + "requires": { + "emoji-regex": "^7.0.1", + "is-fullwidth-code-point": "^2.0.0", + "strip-ansi": "^5.1.0" + } + } + } + }, + "yargs-parser": { + "version": "13.1.2", + "resolved": "https://registry.npmjs.org/yargs-parser/-/yargs-parser-13.1.2.tgz", + "integrity": "sha512-3lbsNRf/j+A4QuSZfDRA7HRSfWrzO0YjqTJd5kjAq37Zep1CEgaYmrH9Q3GwPiB9cHyd1Y1UwggGhJGoxipbzg==", + "dev": true, + "requires": { + "camelcase": "^5.0.0", + "decamelize": "^1.2.0" + } + }, "yn": { "version": "3.1.1", "resolved": "https://registry.npmjs.org/yn/-/yn-3.1.1.tgz", diff --git a/services/real-time/package.json b/services/real-time/package.json index b6b2cfb8b1..5d9f58079e 100644 --- a/services/real-time/package.json +++ b/services/real-time/package.json @@ -8,17 +8,15 @@ "url": "https://github.com/sharelatex/real-time-sharelatex.git" }, "scripts": { - "compile:app": "([ -e app/coffee ] && coffee -m $COFFEE_OPTIONS -o app/js -c app/coffee || echo 'No CoffeeScript folder to compile') && ( [ -e app.coffee ] && coffee -m $COFFEE_OPTIONS -c app.coffee || echo 'No CoffeeScript app to compile')", - "start": "npm run compile:app && node $NODE_APP_OPTIONS app.js", - "test:acceptance:_run": "mocha --recursive --reporter spec --timeout 30000 --exit $@ test/acceptance/js", - "test:acceptance": "npm run compile:app && npm run compile:acceptance_tests && npm run test:acceptance:_run -- --grep=$MOCHA_GREP", - "test:unit:_run": "mocha --recursive --reporter spec --exit $@ test/unit/js", - "test:unit": "npm run compile:app && npm run compile:unit_tests && npm run test:unit:_run -- --grep=$MOCHA_GREP", - "compile:unit_tests": "[ ! -e test/unit/coffee ] && echo 'No unit tests to compile' || coffee -o test/unit/js -c test/unit/coffee", - "compile:acceptance_tests": "[ ! -e test/acceptance/coffee ] && echo 'No acceptance tests to compile' || coffee -o test/acceptance/js -c test/acceptance/coffee", - "compile:all": "npm run compile:app && npm run compile:unit_tests && npm run compile:acceptance_tests && npm run compile:smoke_tests", + "start": "node $NODE_APP_OPTIONS app.js", + "test:acceptance:_run": "mocha --recursive --reporter spec --timeout 15000 --exit $@ test/acceptance/js", + "test:acceptance": "npm run test:acceptance:_run -- --grep=$MOCHA_GREP", + "test:unit:_run": "mocha --recursive --reporter spec $@ test/unit/js", + "test:unit": "npm run test:unit:_run -- --grep=$MOCHA_GREP", "nodemon": "nodemon --config nodemon.json", - "compile:smoke_tests": "[ ! -e test/smoke/coffee ] && echo 'No smoke tests to compile' || coffee -o test/smoke/js -c test/smoke/coffee" + "lint": "node_modules/.bin/eslint .", + "format": "node_modules/.bin/prettier-eslint $PWD'/**/*.js' --list-different", + "format:fix": "node_modules/.bin/prettier-eslint $PWD'/**/*.js' --write" }, "dependencies": { "async": "^0.9.0", @@ -41,10 +39,23 @@ "bunyan": "~0.22.3", "chai": "~1.9.1", "cookie-signature": "^1.1.0", + "eslint": "^6.8.0", + "eslint-config-prettier": "^6.10.0", + "eslint-config-standard": "^14.1.0", + "eslint-plugin-chai-expect": "^2.1.0", + "eslint-plugin-chai-friendly": "^0.5.0", + "eslint-plugin-import": "^2.20.1", + "eslint-plugin-mocha": "^6.3.0", + "eslint-plugin-node": "^11.0.0", + "eslint-plugin-prettier": "^3.1.2", + "eslint-plugin-promise": "^4.2.1", + "eslint-plugin-standard": "^4.0.1", + "mocha": "^4.0.1", + "prettier": "^2.0.0", + "prettier-eslint-cli": "^5.0.0", "sandboxed-module": "~0.3.0", "sinon": "^2.4.1", - "mocha": "^4.0.1", - "uid-safe": "^2.1.5", - "timekeeper": "0.0.4" + "timekeeper": "0.0.4", + "uid-safe": "^2.1.5" } } From 20bb3540e7d67b28e284079b5b86249bb6b10413 Mon Sep 17 00:00:00 2001 From: decaffeinate Date: Tue, 23 Jun 2020 18:29:21 +0100 Subject: [PATCH 367/491] decaffeinate: update .gitignore --- services/real-time/.gitignore | 5 ----- 1 file changed, 5 deletions(-) diff --git a/services/real-time/.gitignore b/services/real-time/.gitignore index ff0c7e15d2..50678c09e9 100644 --- a/services/real-time/.gitignore +++ b/services/real-time/.gitignore @@ -1,7 +1,2 @@ node_modules forever -app.js -app/js -test/unit/js -test/acceptance/js -**/*.map From 90eafa388a01be46390421c9b2dffc25a21ab6f0 Mon Sep 17 00:00:00 2001 From: decaffeinate Date: Tue, 23 Jun 2020 18:29:29 +0100 Subject: [PATCH 368/491] decaffeinate: Rename AuthorizationManager.coffee and 18 other files from .coffee to .js --- .../{AuthorizationManager.coffee => AuthorizationManager.js} | 0 .../app/coffee/{ChannelManager.coffee => ChannelManager.js} | 0 .../{ConnectedUsersManager.coffee => ConnectedUsersManager.js} | 0 ...umentUpdaterController.coffee => DocumentUpdaterController.js} | 0 .../{DocumentUpdaterManager.coffee => DocumentUpdaterManager.js} | 0 .../real-time/app/coffee/{DrainManager.coffee => DrainManager.js} | 0 services/real-time/app/coffee/{Errors.coffee => Errors.js} | 0 .../real-time/app/coffee/{EventLogger.coffee => EventLogger.js} | 0 .../coffee/{HealthCheckManager.coffee => HealthCheckManager.js} | 0 .../app/coffee/{HttpApiController.coffee => HttpApiController.js} | 0 .../app/coffee/{HttpController.coffee => HttpController.js} | 0 .../coffee/{RedisClientManager.coffee => RedisClientManager.js} | 0 .../real-time/app/coffee/{RoomManager.coffee => RoomManager.js} | 0 services/real-time/app/coffee/{Router.coffee => Router.js} | 0 .../app/coffee/{SafeJsonParse.coffee => SafeJsonParse.js} | 0 .../app/coffee/{SessionSockets.coffee => SessionSockets.js} | 0 .../app/coffee/{WebApiManager.coffee => WebApiManager.js} | 0 .../coffee/{WebsocketController.coffee => WebsocketController.js} | 0 .../{WebsocketLoadBalancer.coffee => WebsocketLoadBalancer.js} | 0 19 files changed, 0 insertions(+), 0 deletions(-) rename services/real-time/app/coffee/{AuthorizationManager.coffee => AuthorizationManager.js} (100%) rename services/real-time/app/coffee/{ChannelManager.coffee => ChannelManager.js} (100%) rename services/real-time/app/coffee/{ConnectedUsersManager.coffee => ConnectedUsersManager.js} (100%) rename services/real-time/app/coffee/{DocumentUpdaterController.coffee => DocumentUpdaterController.js} (100%) rename services/real-time/app/coffee/{DocumentUpdaterManager.coffee => DocumentUpdaterManager.js} (100%) rename services/real-time/app/coffee/{DrainManager.coffee => DrainManager.js} (100%) rename services/real-time/app/coffee/{Errors.coffee => Errors.js} (100%) rename services/real-time/app/coffee/{EventLogger.coffee => EventLogger.js} (100%) rename services/real-time/app/coffee/{HealthCheckManager.coffee => HealthCheckManager.js} (100%) rename services/real-time/app/coffee/{HttpApiController.coffee => HttpApiController.js} (100%) rename services/real-time/app/coffee/{HttpController.coffee => HttpController.js} (100%) rename services/real-time/app/coffee/{RedisClientManager.coffee => RedisClientManager.js} (100%) rename services/real-time/app/coffee/{RoomManager.coffee => RoomManager.js} (100%) rename services/real-time/app/coffee/{Router.coffee => Router.js} (100%) rename services/real-time/app/coffee/{SafeJsonParse.coffee => SafeJsonParse.js} (100%) rename services/real-time/app/coffee/{SessionSockets.coffee => SessionSockets.js} (100%) rename services/real-time/app/coffee/{WebApiManager.coffee => WebApiManager.js} (100%) rename services/real-time/app/coffee/{WebsocketController.coffee => WebsocketController.js} (100%) rename services/real-time/app/coffee/{WebsocketLoadBalancer.coffee => WebsocketLoadBalancer.js} (100%) diff --git a/services/real-time/app/coffee/AuthorizationManager.coffee b/services/real-time/app/coffee/AuthorizationManager.js similarity index 100% rename from services/real-time/app/coffee/AuthorizationManager.coffee rename to services/real-time/app/coffee/AuthorizationManager.js diff --git a/services/real-time/app/coffee/ChannelManager.coffee b/services/real-time/app/coffee/ChannelManager.js similarity index 100% rename from services/real-time/app/coffee/ChannelManager.coffee rename to services/real-time/app/coffee/ChannelManager.js diff --git a/services/real-time/app/coffee/ConnectedUsersManager.coffee b/services/real-time/app/coffee/ConnectedUsersManager.js similarity index 100% rename from services/real-time/app/coffee/ConnectedUsersManager.coffee rename to services/real-time/app/coffee/ConnectedUsersManager.js diff --git a/services/real-time/app/coffee/DocumentUpdaterController.coffee b/services/real-time/app/coffee/DocumentUpdaterController.js similarity index 100% rename from services/real-time/app/coffee/DocumentUpdaterController.coffee rename to services/real-time/app/coffee/DocumentUpdaterController.js diff --git a/services/real-time/app/coffee/DocumentUpdaterManager.coffee b/services/real-time/app/coffee/DocumentUpdaterManager.js similarity index 100% rename from services/real-time/app/coffee/DocumentUpdaterManager.coffee rename to services/real-time/app/coffee/DocumentUpdaterManager.js diff --git a/services/real-time/app/coffee/DrainManager.coffee b/services/real-time/app/coffee/DrainManager.js similarity index 100% rename from services/real-time/app/coffee/DrainManager.coffee rename to services/real-time/app/coffee/DrainManager.js diff --git a/services/real-time/app/coffee/Errors.coffee b/services/real-time/app/coffee/Errors.js similarity index 100% rename from services/real-time/app/coffee/Errors.coffee rename to services/real-time/app/coffee/Errors.js diff --git a/services/real-time/app/coffee/EventLogger.coffee b/services/real-time/app/coffee/EventLogger.js similarity index 100% rename from services/real-time/app/coffee/EventLogger.coffee rename to services/real-time/app/coffee/EventLogger.js diff --git a/services/real-time/app/coffee/HealthCheckManager.coffee b/services/real-time/app/coffee/HealthCheckManager.js similarity index 100% rename from services/real-time/app/coffee/HealthCheckManager.coffee rename to services/real-time/app/coffee/HealthCheckManager.js diff --git a/services/real-time/app/coffee/HttpApiController.coffee b/services/real-time/app/coffee/HttpApiController.js similarity index 100% rename from services/real-time/app/coffee/HttpApiController.coffee rename to services/real-time/app/coffee/HttpApiController.js diff --git a/services/real-time/app/coffee/HttpController.coffee b/services/real-time/app/coffee/HttpController.js similarity index 100% rename from services/real-time/app/coffee/HttpController.coffee rename to services/real-time/app/coffee/HttpController.js diff --git a/services/real-time/app/coffee/RedisClientManager.coffee b/services/real-time/app/coffee/RedisClientManager.js similarity index 100% rename from services/real-time/app/coffee/RedisClientManager.coffee rename to services/real-time/app/coffee/RedisClientManager.js diff --git a/services/real-time/app/coffee/RoomManager.coffee b/services/real-time/app/coffee/RoomManager.js similarity index 100% rename from services/real-time/app/coffee/RoomManager.coffee rename to services/real-time/app/coffee/RoomManager.js diff --git a/services/real-time/app/coffee/Router.coffee b/services/real-time/app/coffee/Router.js similarity index 100% rename from services/real-time/app/coffee/Router.coffee rename to services/real-time/app/coffee/Router.js diff --git a/services/real-time/app/coffee/SafeJsonParse.coffee b/services/real-time/app/coffee/SafeJsonParse.js similarity index 100% rename from services/real-time/app/coffee/SafeJsonParse.coffee rename to services/real-time/app/coffee/SafeJsonParse.js diff --git a/services/real-time/app/coffee/SessionSockets.coffee b/services/real-time/app/coffee/SessionSockets.js similarity index 100% rename from services/real-time/app/coffee/SessionSockets.coffee rename to services/real-time/app/coffee/SessionSockets.js diff --git a/services/real-time/app/coffee/WebApiManager.coffee b/services/real-time/app/coffee/WebApiManager.js similarity index 100% rename from services/real-time/app/coffee/WebApiManager.coffee rename to services/real-time/app/coffee/WebApiManager.js diff --git a/services/real-time/app/coffee/WebsocketController.coffee b/services/real-time/app/coffee/WebsocketController.js similarity index 100% rename from services/real-time/app/coffee/WebsocketController.coffee rename to services/real-time/app/coffee/WebsocketController.js diff --git a/services/real-time/app/coffee/WebsocketLoadBalancer.coffee b/services/real-time/app/coffee/WebsocketLoadBalancer.js similarity index 100% rename from services/real-time/app/coffee/WebsocketLoadBalancer.coffee rename to services/real-time/app/coffee/WebsocketLoadBalancer.js From 7335084c26da73c142c594daa37cfc34774ae9e9 Mon Sep 17 00:00:00 2001 From: decaffeinate Date: Tue, 23 Jun 2020 18:29:34 +0100 Subject: [PATCH 369/491] decaffeinate: Convert AuthorizationManager.coffee and 18 other files to JS --- .../app/coffee/AuthorizationManager.js | 87 ++- .../real-time/app/coffee/ChannelManager.js | 141 +++-- .../app/coffee/ConnectedUsersManager.js | 178 +++--- .../app/coffee/DocumentUpdaterController.js | 206 ++++--- .../app/coffee/DocumentUpdaterManager.js | 176 +++--- services/real-time/app/coffee/DrainManager.js | 88 +-- services/real-time/app/coffee/Errors.js | 20 +- services/real-time/app/coffee/EventLogger.js | 130 +++-- .../app/coffee/HealthCheckManager.js | 119 ++-- .../real-time/app/coffee/HttpApiController.js | 77 ++- .../real-time/app/coffee/HttpController.js | 82 ++- .../app/coffee/RedisClientManager.js | 51 +- services/real-time/app/coffee/RoomManager.js | 234 +++++--- services/real-time/app/coffee/Router.js | 394 ++++++++----- .../real-time/app/coffee/SafeJsonParse.js | 36 +- .../real-time/app/coffee/SessionSockets.js | 49 +- .../real-time/app/coffee/WebApiManager.js | 88 +-- .../app/coffee/WebsocketController.js | 550 ++++++++++-------- .../app/coffee/WebsocketLoadBalancer.js | 213 ++++--- 19 files changed, 1732 insertions(+), 1187 deletions(-) diff --git a/services/real-time/app/coffee/AuthorizationManager.js b/services/real-time/app/coffee/AuthorizationManager.js index 50d76537ce..0ce4c313e2 100644 --- a/services/real-time/app/coffee/AuthorizationManager.js +++ b/services/real-time/app/coffee/AuthorizationManager.js @@ -1,36 +1,65 @@ -module.exports = AuthorizationManager = - assertClientCanViewProject: (client, callback = (error) ->) -> - AuthorizationManager._assertClientHasPrivilegeLevel client, ["readOnly", "readAndWrite", "owner"], callback +/* + * decaffeinate suggestions: + * DS101: Remove unnecessary use of Array.from + * DS102: Remove unnecessary code created because of implicit returns + * DS207: Consider shorter variations of null checks + * Full docs: https://github.com/decaffeinate/decaffeinate/blob/master/docs/suggestions.md + */ +let AuthorizationManager; +module.exports = (AuthorizationManager = { + assertClientCanViewProject(client, callback) { + if (callback == null) { callback = function(error) {}; } + return AuthorizationManager._assertClientHasPrivilegeLevel(client, ["readOnly", "readAndWrite", "owner"], callback); + }, - assertClientCanEditProject: (client, callback = (error) ->) -> - AuthorizationManager._assertClientHasPrivilegeLevel client, ["readAndWrite", "owner"], callback + assertClientCanEditProject(client, callback) { + if (callback == null) { callback = function(error) {}; } + return AuthorizationManager._assertClientHasPrivilegeLevel(client, ["readAndWrite", "owner"], callback); + }, - _assertClientHasPrivilegeLevel: (client, allowedLevels, callback = (error) ->) -> - if client.ol_context["privilege_level"] in allowedLevels - callback null - else - callback new Error("not authorized") + _assertClientHasPrivilegeLevel(client, allowedLevels, callback) { + if (callback == null) { callback = function(error) {}; } + if (Array.from(allowedLevels).includes(client.ol_context["privilege_level"])) { + return callback(null); + } else { + return callback(new Error("not authorized")); + } + }, - assertClientCanViewProjectAndDoc: (client, doc_id, callback = (error) ->) -> - AuthorizationManager.assertClientCanViewProject client, (error) -> - return callback(error) if error? - AuthorizationManager._assertClientCanAccessDoc client, doc_id, callback + assertClientCanViewProjectAndDoc(client, doc_id, callback) { + if (callback == null) { callback = function(error) {}; } + return AuthorizationManager.assertClientCanViewProject(client, function(error) { + if (error != null) { return callback(error); } + return AuthorizationManager._assertClientCanAccessDoc(client, doc_id, callback); + }); + }, - assertClientCanEditProjectAndDoc: (client, doc_id, callback = (error) ->) -> - AuthorizationManager.assertClientCanEditProject client, (error) -> - return callback(error) if error? - AuthorizationManager._assertClientCanAccessDoc client, doc_id, callback + assertClientCanEditProjectAndDoc(client, doc_id, callback) { + if (callback == null) { callback = function(error) {}; } + return AuthorizationManager.assertClientCanEditProject(client, function(error) { + if (error != null) { return callback(error); } + return AuthorizationManager._assertClientCanAccessDoc(client, doc_id, callback); + }); + }, - _assertClientCanAccessDoc: (client, doc_id, callback = (error) ->) -> - if client.ol_context["doc:#{doc_id}"] is "allowed" - callback null - else - callback new Error("not authorized") + _assertClientCanAccessDoc(client, doc_id, callback) { + if (callback == null) { callback = function(error) {}; } + if (client.ol_context[`doc:${doc_id}`] === "allowed") { + return callback(null); + } else { + return callback(new Error("not authorized")); + } + }, - addAccessToDoc: (client, doc_id, callback = (error) ->) -> - client.ol_context["doc:#{doc_id}"] = "allowed" - callback(null) + addAccessToDoc(client, doc_id, callback) { + if (callback == null) { callback = function(error) {}; } + client.ol_context[`doc:${doc_id}`] = "allowed"; + return callback(null); + }, - removeAccessToDoc: (client, doc_id, callback = (error) ->) -> - delete client.ol_context["doc:#{doc_id}"] - callback(null) + removeAccessToDoc(client, doc_id, callback) { + if (callback == null) { callback = function(error) {}; } + delete client.ol_context[`doc:${doc_id}`]; + return callback(null); + } +}); diff --git a/services/real-time/app/coffee/ChannelManager.js b/services/real-time/app/coffee/ChannelManager.js index e60a145bd5..eb73802a07 100644 --- a/services/real-time/app/coffee/ChannelManager.js +++ b/services/real-time/app/coffee/ChannelManager.js @@ -1,71 +1,86 @@ -logger = require 'logger-sharelatex' -metrics = require "metrics-sharelatex" -settings = require "settings-sharelatex" +/* + * decaffeinate suggestions: + * DS102: Remove unnecessary code created because of implicit returns + * Full docs: https://github.com/decaffeinate/decaffeinate/blob/master/docs/suggestions.md + */ +let ChannelManager; +const logger = require('logger-sharelatex'); +const metrics = require("metrics-sharelatex"); +const settings = require("settings-sharelatex"); -ClientMap = new Map() # for each redis client, store a Map of subscribed channels (channelname -> subscribe promise) +const ClientMap = new Map(); // for each redis client, store a Map of subscribed channels (channelname -> subscribe promise) -# Manage redis pubsub subscriptions for individual projects and docs, ensuring -# that we never subscribe to a channel multiple times. The socket.io side is -# handled by RoomManager. +// Manage redis pubsub subscriptions for individual projects and docs, ensuring +// that we never subscribe to a channel multiple times. The socket.io side is +// handled by RoomManager. -module.exports = ChannelManager = - getClientMapEntry: (rclient) -> - # return the per-client channel map if it exists, otherwise create and - # return an empty map for the client. - ClientMap.get(rclient) || ClientMap.set(rclient, new Map()).get(rclient) +module.exports = (ChannelManager = { + getClientMapEntry(rclient) { + // return the per-client channel map if it exists, otherwise create and + // return an empty map for the client. + return ClientMap.get(rclient) || ClientMap.set(rclient, new Map()).get(rclient); + }, - subscribe: (rclient, baseChannel, id) -> - clientChannelMap = @getClientMapEntry(rclient) - channel = "#{baseChannel}:#{id}" - actualSubscribe = () -> - # subscribe is happening in the foreground and it should reject - p = rclient.subscribe(channel) - p.finally () -> - if clientChannelMap.get(channel) is subscribePromise - clientChannelMap.delete(channel) - .then () -> - logger.log {channel}, "subscribed to channel" - metrics.inc "subscribe.#{baseChannel}" - .catch (err) -> - logger.error {channel, err}, "failed to subscribe to channel" - metrics.inc "subscribe.failed.#{baseChannel}" - return p + subscribe(rclient, baseChannel, id) { + const clientChannelMap = this.getClientMapEntry(rclient); + const channel = `${baseChannel}:${id}`; + const actualSubscribe = function() { + // subscribe is happening in the foreground and it should reject + const p = rclient.subscribe(channel); + p.finally(function() { + if (clientChannelMap.get(channel) === subscribePromise) { + return clientChannelMap.delete(channel); + }}).then(function() { + logger.log({channel}, "subscribed to channel"); + return metrics.inc(`subscribe.${baseChannel}`);}).catch(function(err) { + logger.error({channel, err}, "failed to subscribe to channel"); + return metrics.inc(`subscribe.failed.${baseChannel}`); + }); + return p; + }; - pendingActions = clientChannelMap.get(channel) || Promise.resolve() - subscribePromise = pendingActions.then(actualSubscribe, actualSubscribe) - clientChannelMap.set(channel, subscribePromise) - logger.log {channel}, "planned to subscribe to channel" - return subscribePromise + const pendingActions = clientChannelMap.get(channel) || Promise.resolve(); + var subscribePromise = pendingActions.then(actualSubscribe, actualSubscribe); + clientChannelMap.set(channel, subscribePromise); + logger.log({channel}, "planned to subscribe to channel"); + return subscribePromise; + }, - unsubscribe: (rclient, baseChannel, id) -> - clientChannelMap = @getClientMapEntry(rclient) - channel = "#{baseChannel}:#{id}" - actualUnsubscribe = () -> - # unsubscribe is happening in the background, it should not reject - p = rclient.unsubscribe(channel) - .finally () -> - if clientChannelMap.get(channel) is unsubscribePromise - clientChannelMap.delete(channel) - .then () -> - logger.log {channel}, "unsubscribed from channel" - metrics.inc "unsubscribe.#{baseChannel}" - .catch (err) -> - logger.error {channel, err}, "unsubscribed from channel" - metrics.inc "unsubscribe.failed.#{baseChannel}" - return p + unsubscribe(rclient, baseChannel, id) { + const clientChannelMap = this.getClientMapEntry(rclient); + const channel = `${baseChannel}:${id}`; + const actualUnsubscribe = function() { + // unsubscribe is happening in the background, it should not reject + const p = rclient.unsubscribe(channel) + .finally(function() { + if (clientChannelMap.get(channel) === unsubscribePromise) { + return clientChannelMap.delete(channel); + }}).then(function() { + logger.log({channel}, "unsubscribed from channel"); + return metrics.inc(`unsubscribe.${baseChannel}`);}).catch(function(err) { + logger.error({channel, err}, "unsubscribed from channel"); + return metrics.inc(`unsubscribe.failed.${baseChannel}`); + }); + return p; + }; - pendingActions = clientChannelMap.get(channel) || Promise.resolve() - unsubscribePromise = pendingActions.then(actualUnsubscribe, actualUnsubscribe) - clientChannelMap.set(channel, unsubscribePromise) - logger.log {channel}, "planned to unsubscribe from channel" - return unsubscribePromise + const pendingActions = clientChannelMap.get(channel) || Promise.resolve(); + var unsubscribePromise = pendingActions.then(actualUnsubscribe, actualUnsubscribe); + clientChannelMap.set(channel, unsubscribePromise); + logger.log({channel}, "planned to unsubscribe from channel"); + return unsubscribePromise; + }, - publish: (rclient, baseChannel, id, data) -> - metrics.summary "redis.publish.#{baseChannel}", data.length - if id is 'all' or !settings.publishOnIndividualChannels - channel = baseChannel - else - channel = "#{baseChannel}:#{id}" - # we publish on a different client to the subscribe, so we can't - # check for the channel existing here - rclient.publish channel, data + publish(rclient, baseChannel, id, data) { + let channel; + metrics.summary(`redis.publish.${baseChannel}`, data.length); + if ((id === 'all') || !settings.publishOnIndividualChannels) { + channel = baseChannel; + } else { + channel = `${baseChannel}:${id}`; + } + // we publish on a different client to the subscribe, so we can't + // check for the channel existing here + return rclient.publish(channel, data); + } +}); diff --git a/services/real-time/app/coffee/ConnectedUsersManager.js b/services/real-time/app/coffee/ConnectedUsersManager.js index 2e6536c9be..bfdcf608a0 100644 --- a/services/real-time/app/coffee/ConnectedUsersManager.js +++ b/services/real-time/app/coffee/ConnectedUsersManager.js @@ -1,91 +1,115 @@ -async = require("async") -Settings = require('settings-sharelatex') -logger = require("logger-sharelatex") -redis = require("redis-sharelatex") -rclient = redis.createClient(Settings.redis.realtime) -Keys = Settings.redis.realtime.key_schema +/* + * decaffeinate suggestions: + * DS102: Remove unnecessary code created because of implicit returns + * DS207: Consider shorter variations of null checks + * Full docs: https://github.com/decaffeinate/decaffeinate/blob/master/docs/suggestions.md + */ +const async = require("async"); +const Settings = require('settings-sharelatex'); +const logger = require("logger-sharelatex"); +const redis = require("redis-sharelatex"); +const rclient = redis.createClient(Settings.redis.realtime); +const Keys = Settings.redis.realtime.key_schema; -ONE_HOUR_IN_S = 60 * 60 -ONE_DAY_IN_S = ONE_HOUR_IN_S * 24 -FOUR_DAYS_IN_S = ONE_DAY_IN_S * 4 +const ONE_HOUR_IN_S = 60 * 60; +const ONE_DAY_IN_S = ONE_HOUR_IN_S * 24; +const FOUR_DAYS_IN_S = ONE_DAY_IN_S * 4; -USER_TIMEOUT_IN_S = ONE_HOUR_IN_S / 4 -REFRESH_TIMEOUT_IN_S = 10 # only show clients which have responded to a refresh request in the last 10 seconds +const USER_TIMEOUT_IN_S = ONE_HOUR_IN_S / 4; +const REFRESH_TIMEOUT_IN_S = 10; // only show clients which have responded to a refresh request in the last 10 seconds -module.exports = - # Use the same method for when a user connects, and when a user sends a cursor - # update. This way we don't care if the connected_user key has expired when - # we receive a cursor update. - updateUserPosition: (project_id, client_id, user, cursorData, callback = (err)->)-> - logger.log project_id:project_id, client_id:client_id, "marking user as joined or connected" +module.exports = { + // Use the same method for when a user connects, and when a user sends a cursor + // update. This way we don't care if the connected_user key has expired when + // we receive a cursor update. + updateUserPosition(project_id, client_id, user, cursorData, callback){ + if (callback == null) { callback = function(err){}; } + logger.log({project_id, client_id}, "marking user as joined or connected"); - multi = rclient.multi() + const multi = rclient.multi(); - multi.sadd Keys.clientsInProject({project_id}), client_id - multi.expire Keys.clientsInProject({project_id}), FOUR_DAYS_IN_S + multi.sadd(Keys.clientsInProject({project_id}), client_id); + multi.expire(Keys.clientsInProject({project_id}), FOUR_DAYS_IN_S); - multi.hset Keys.connectedUser({project_id, client_id}), "last_updated_at", Date.now() - multi.hset Keys.connectedUser({project_id, client_id}), "user_id", user._id - multi.hset Keys.connectedUser({project_id, client_id}), "first_name", user.first_name or "" - multi.hset Keys.connectedUser({project_id, client_id}), "last_name", user.last_name or "" - multi.hset Keys.connectedUser({project_id, client_id}), "email", user.email or "" + multi.hset(Keys.connectedUser({project_id, client_id}), "last_updated_at", Date.now()); + multi.hset(Keys.connectedUser({project_id, client_id}), "user_id", user._id); + multi.hset(Keys.connectedUser({project_id, client_id}), "first_name", user.first_name || ""); + multi.hset(Keys.connectedUser({project_id, client_id}), "last_name", user.last_name || ""); + multi.hset(Keys.connectedUser({project_id, client_id}), "email", user.email || ""); - if cursorData? - multi.hset Keys.connectedUser({project_id, client_id}), "cursorData", JSON.stringify(cursorData) - multi.expire Keys.connectedUser({project_id, client_id}), USER_TIMEOUT_IN_S + if (cursorData != null) { + multi.hset(Keys.connectedUser({project_id, client_id}), "cursorData", JSON.stringify(cursorData)); + } + multi.expire(Keys.connectedUser({project_id, client_id}), USER_TIMEOUT_IN_S); - multi.exec (err)-> - if err? - logger.err err:err, project_id:project_id, client_id:client_id, "problem marking user as connected" - callback(err) + return multi.exec(function(err){ + if (err != null) { + logger.err({err, project_id, client_id}, "problem marking user as connected"); + } + return callback(err); + }); + }, - refreshClient: (project_id, client_id, callback = (err) ->) -> - logger.log project_id:project_id, client_id:client_id, "refreshing connected client" - multi = rclient.multi() - multi.hset Keys.connectedUser({project_id, client_id}), "last_updated_at", Date.now() - multi.expire Keys.connectedUser({project_id, client_id}), USER_TIMEOUT_IN_S - multi.exec (err)-> - if err? - logger.err err:err, project_id:project_id, client_id:client_id, "problem refreshing connected client" - callback(err) + refreshClient(project_id, client_id, callback) { + if (callback == null) { callback = function(err) {}; } + logger.log({project_id, client_id}, "refreshing connected client"); + const multi = rclient.multi(); + multi.hset(Keys.connectedUser({project_id, client_id}), "last_updated_at", Date.now()); + multi.expire(Keys.connectedUser({project_id, client_id}), USER_TIMEOUT_IN_S); + return multi.exec(function(err){ + if (err != null) { + logger.err({err, project_id, client_id}, "problem refreshing connected client"); + } + return callback(err); + }); + }, - markUserAsDisconnected: (project_id, client_id, callback)-> - logger.log project_id:project_id, client_id:client_id, "marking user as disconnected" - multi = rclient.multi() - multi.srem Keys.clientsInProject({project_id}), client_id - multi.expire Keys.clientsInProject({project_id}), FOUR_DAYS_IN_S - multi.del Keys.connectedUser({project_id, client_id}) - multi.exec callback + markUserAsDisconnected(project_id, client_id, callback){ + logger.log({project_id, client_id}, "marking user as disconnected"); + const multi = rclient.multi(); + multi.srem(Keys.clientsInProject({project_id}), client_id); + multi.expire(Keys.clientsInProject({project_id}), FOUR_DAYS_IN_S); + multi.del(Keys.connectedUser({project_id, client_id})); + return multi.exec(callback); + }, - _getConnectedUser: (project_id, client_id, callback)-> - rclient.hgetall Keys.connectedUser({project_id, client_id}), (err, result)-> - if !result? or Object.keys(result).length == 0 or !result.user_id - result = - connected : false - client_id:client_id - else - result.connected = true - result.client_id = client_id - result.client_age = (Date.now() - parseInt(result.last_updated_at,10)) / 1000 - if result.cursorData? - try - result.cursorData = JSON.parse(result.cursorData) - catch e - logger.error {err: e, project_id, client_id, cursorData: result.cursorData}, "error parsing cursorData JSON" - return callback e - callback err, result + _getConnectedUser(project_id, client_id, callback){ + return rclient.hgetall(Keys.connectedUser({project_id, client_id}), function(err, result){ + if ((result == null) || (Object.keys(result).length === 0) || !result.user_id) { + result = { + connected : false, + client_id + }; + } else { + result.connected = true; + result.client_id = client_id; + result.client_age = (Date.now() - parseInt(result.last_updated_at,10)) / 1000; + if (result.cursorData != null) { + try { + result.cursorData = JSON.parse(result.cursorData); + } catch (e) { + logger.error({err: e, project_id, client_id, cursorData: result.cursorData}, "error parsing cursorData JSON"); + return callback(e); + } + } + } + return callback(err, result); + }); + }, - getConnectedUsers: (project_id, callback)-> - self = @ - rclient.smembers Keys.clientsInProject({project_id}), (err, results)-> - return callback(err) if err? - jobs = results.map (client_id)-> - (cb)-> - self._getConnectedUser(project_id, client_id, cb) - async.series jobs, (err, users = [])-> - return callback(err) if err? - users = users.filter (user) -> - user?.connected && user?.client_age < REFRESH_TIMEOUT_IN_S - callback null, users + getConnectedUsers(project_id, callback){ + const self = this; + return rclient.smembers(Keys.clientsInProject({project_id}), function(err, results){ + if (err != null) { return callback(err); } + const jobs = results.map(client_id => cb => self._getConnectedUser(project_id, client_id, cb)); + return async.series(jobs, function(err, users){ + if (users == null) { users = []; } + if (err != null) { return callback(err); } + users = users.filter(user => (user != null ? user.connected : undefined) && ((user != null ? user.client_age : undefined) < REFRESH_TIMEOUT_IN_S)); + return callback(null, users); + }); + }); + } +}; diff --git a/services/real-time/app/coffee/DocumentUpdaterController.js b/services/real-time/app/coffee/DocumentUpdaterController.js index 24b6a7c525..85078219b6 100644 --- a/services/real-time/app/coffee/DocumentUpdaterController.js +++ b/services/real-time/app/coffee/DocumentUpdaterController.js @@ -1,88 +1,136 @@ -logger = require "logger-sharelatex" -settings = require 'settings-sharelatex' -RedisClientManager = require "./RedisClientManager" -SafeJsonParse = require "./SafeJsonParse" -EventLogger = require "./EventLogger" -HealthCheckManager = require "./HealthCheckManager" -RoomManager = require "./RoomManager" -ChannelManager = require "./ChannelManager" -metrics = require "metrics-sharelatex" +/* + * decaffeinate suggestions: + * DS101: Remove unnecessary use of Array.from + * DS102: Remove unnecessary code created because of implicit returns + * DS205: Consider reworking code to avoid use of IIFEs + * DS207: Consider shorter variations of null checks + * Full docs: https://github.com/decaffeinate/decaffeinate/blob/master/docs/suggestions.md + */ +let DocumentUpdaterController; +const logger = require("logger-sharelatex"); +const settings = require('settings-sharelatex'); +const RedisClientManager = require("./RedisClientManager"); +const SafeJsonParse = require("./SafeJsonParse"); +const EventLogger = require("./EventLogger"); +const HealthCheckManager = require("./HealthCheckManager"); +const RoomManager = require("./RoomManager"); +const ChannelManager = require("./ChannelManager"); +const metrics = require("metrics-sharelatex"); -MESSAGE_SIZE_LOG_LIMIT = 1024 * 1024 # 1Mb +const MESSAGE_SIZE_LOG_LIMIT = 1024 * 1024; // 1Mb -module.exports = DocumentUpdaterController = - # DocumentUpdaterController is responsible for updates that come via Redis - # Pub/Sub from the document updater. - rclientList: RedisClientManager.createClientList(settings.redis.pubsub) +module.exports = (DocumentUpdaterController = { + // DocumentUpdaterController is responsible for updates that come via Redis + // Pub/Sub from the document updater. + rclientList: RedisClientManager.createClientList(settings.redis.pubsub), - listenForUpdatesFromDocumentUpdater: (io) -> - logger.log {rclients: @rclientList.length}, "listening for applied-ops events" - for rclient, i in @rclientList - rclient.subscribe "applied-ops" - rclient.on "message", (channel, message) -> - metrics.inc "rclient", 0.001 # global event rate metric - EventLogger.debugEvent(channel, message) if settings.debugEvents > 0 - DocumentUpdaterController._processMessageFromDocumentUpdater(io, channel, message) - # create metrics for each redis instance only when we have multiple redis clients - if @rclientList.length > 1 - for rclient, i in @rclientList - do (i) -> - rclient.on "message", () -> - metrics.inc "rclient-#{i}", 0.001 # per client event rate metric - @handleRoomUpdates(@rclientList) + listenForUpdatesFromDocumentUpdater(io) { + let i, rclient; + logger.log({rclients: this.rclientList.length}, "listening for applied-ops events"); + for (i = 0; i < this.rclientList.length; i++) { + rclient = this.rclientList[i]; + rclient.subscribe("applied-ops"); + rclient.on("message", function(channel, message) { + metrics.inc("rclient", 0.001); // global event rate metric + if (settings.debugEvents > 0) { EventLogger.debugEvent(channel, message); } + return DocumentUpdaterController._processMessageFromDocumentUpdater(io, channel, message); + }); + } + // create metrics for each redis instance only when we have multiple redis clients + if (this.rclientList.length > 1) { + for (i = 0; i < this.rclientList.length; i++) { + rclient = this.rclientList[i]; + ((i => // per client event rate metric + rclient.on("message", () => metrics.inc(`rclient-${i}`, 0.001))))(i); + } + } + return this.handleRoomUpdates(this.rclientList); + }, - handleRoomUpdates: (rclientSubList) -> - roomEvents = RoomManager.eventSource() - roomEvents.on 'doc-active', (doc_id) -> - subscribePromises = for rclient in rclientSubList - ChannelManager.subscribe rclient, "applied-ops", doc_id - RoomManager.emitOnCompletion(subscribePromises, "doc-subscribed-#{doc_id}") - roomEvents.on 'doc-empty', (doc_id) -> - for rclient in rclientSubList - ChannelManager.unsubscribe rclient, "applied-ops", doc_id + handleRoomUpdates(rclientSubList) { + const roomEvents = RoomManager.eventSource(); + roomEvents.on('doc-active', function(doc_id) { + const subscribePromises = Array.from(rclientSubList).map((rclient) => + ChannelManager.subscribe(rclient, "applied-ops", doc_id)); + return RoomManager.emitOnCompletion(subscribePromises, `doc-subscribed-${doc_id}`); + }); + return roomEvents.on('doc-empty', doc_id => Array.from(rclientSubList).map((rclient) => + ChannelManager.unsubscribe(rclient, "applied-ops", doc_id))); + }, - _processMessageFromDocumentUpdater: (io, channel, message) -> - SafeJsonParse.parse message, (error, message) -> - if error? - logger.error {err: error, channel}, "error parsing JSON" - return - if message.op? - if message._id? && settings.checkEventOrder - status = EventLogger.checkEventOrder("applied-ops", message._id, message) - if status is 'duplicate' - return # skip duplicate events - DocumentUpdaterController._applyUpdateFromDocumentUpdater(io, message.doc_id, message.op) - else if message.error? - DocumentUpdaterController._processErrorFromDocumentUpdater(io, message.doc_id, message.error, message) - else if message.health_check? - logger.debug {message}, "got health check message in applied ops channel" - HealthCheckManager.check channel, message.key + _processMessageFromDocumentUpdater(io, channel, message) { + return SafeJsonParse.parse(message, function(error, message) { + if (error != null) { + logger.error({err: error, channel}, "error parsing JSON"); + return; + } + if (message.op != null) { + if ((message._id != null) && settings.checkEventOrder) { + const status = EventLogger.checkEventOrder("applied-ops", message._id, message); + if (status === 'duplicate') { + return; // skip duplicate events + } + } + return DocumentUpdaterController._applyUpdateFromDocumentUpdater(io, message.doc_id, message.op); + } else if (message.error != null) { + return DocumentUpdaterController._processErrorFromDocumentUpdater(io, message.doc_id, message.error, message); + } else if (message.health_check != null) { + logger.debug({message}, "got health check message in applied ops channel"); + return HealthCheckManager.check(channel, message.key); + } + }); + }, - _applyUpdateFromDocumentUpdater: (io, doc_id, update) -> - clientList = io.sockets.clients(doc_id) - # avoid unnecessary work if no clients are connected - if clientList.length is 0 - return - # send updates to clients - logger.log doc_id: doc_id, version: update.v, source: update.meta?.source, socketIoClients: (client.id for client in clientList), "distributing updates to clients" - seen = {} - # send messages only to unique clients (due to duplicate entries in io.sockets.clients) - for client in clientList when not seen[client.id] - seen[client.id] = true - if client.publicId == update.meta.source - logger.log doc_id: doc_id, version: update.v, source: update.meta?.source, "distributing update to sender" - client.emit "otUpdateApplied", v: update.v, doc: update.doc - else if !update.dup # Duplicate ops should just be sent back to sending client for acknowledgement - logger.log doc_id: doc_id, version: update.v, source: update.meta?.source, client_id: client.id, "distributing update to collaborator" - client.emit "otUpdateApplied", update - if Object.keys(seen).length < clientList.length - metrics.inc "socket-io.duplicate-clients", 0.1 - logger.log doc_id: doc_id, socketIoClients: (client.id for client in clientList), "discarded duplicate clients" + _applyUpdateFromDocumentUpdater(io, doc_id, update) { + let client; + const clientList = io.sockets.clients(doc_id); + // avoid unnecessary work if no clients are connected + if (clientList.length === 0) { + return; + } + // send updates to clients + logger.log({doc_id, version: update.v, source: (update.meta != null ? update.meta.source : undefined), socketIoClients: (((() => { + const result = []; + for (client of Array.from(clientList)) { result.push(client.id); + } + return result; + })()))}, "distributing updates to clients"); + const seen = {}; + // send messages only to unique clients (due to duplicate entries in io.sockets.clients) + for (client of Array.from(clientList)) { + if (!seen[client.id]) { + seen[client.id] = true; + if (client.publicId === update.meta.source) { + logger.log({doc_id, version: update.v, source: (update.meta != null ? update.meta.source : undefined)}, "distributing update to sender"); + client.emit("otUpdateApplied", {v: update.v, doc: update.doc}); + } else if (!update.dup) { // Duplicate ops should just be sent back to sending client for acknowledgement + logger.log({doc_id, version: update.v, source: (update.meta != null ? update.meta.source : undefined), client_id: client.id}, "distributing update to collaborator"); + client.emit("otUpdateApplied", update); + } + } + } + if (Object.keys(seen).length < clientList.length) { + metrics.inc("socket-io.duplicate-clients", 0.1); + return logger.log({doc_id, socketIoClients: (((() => { + const result1 = []; + for (client of Array.from(clientList)) { result1.push(client.id); + } + return result1; + })()))}, "discarded duplicate clients"); + } + }, - _processErrorFromDocumentUpdater: (io, doc_id, error, message) -> - for client in io.sockets.clients(doc_id) - logger.warn err: error, doc_id: doc_id, client_id: client.id, "error from document updater, disconnecting client" - client.emit "otUpdateError", error, message - client.disconnect() + _processErrorFromDocumentUpdater(io, doc_id, error, message) { + return (() => { + const result = []; + for (let client of Array.from(io.sockets.clients(doc_id))) { + logger.warn({err: error, doc_id, client_id: client.id}, "error from document updater, disconnecting client"); + client.emit("otUpdateError", error, message); + result.push(client.disconnect()); + } + return result; + })(); + } +}); diff --git a/services/real-time/app/coffee/DocumentUpdaterManager.js b/services/real-time/app/coffee/DocumentUpdaterManager.js index c5c5a67cb7..4b07b8f381 100644 --- a/services/real-time/app/coffee/DocumentUpdaterManager.js +++ b/services/real-time/app/coffee/DocumentUpdaterManager.js @@ -1,83 +1,107 @@ -request = require "request" -_ = require "underscore" -logger = require "logger-sharelatex" -settings = require "settings-sharelatex" -metrics = require("metrics-sharelatex") +/* + * decaffeinate suggestions: + * DS102: Remove unnecessary code created because of implicit returns + * DS207: Consider shorter variations of null checks + * Full docs: https://github.com/decaffeinate/decaffeinate/blob/master/docs/suggestions.md + */ +let DocumentUpdaterManager; +const request = require("request"); +const _ = require("underscore"); +const logger = require("logger-sharelatex"); +const settings = require("settings-sharelatex"); +const metrics = require("metrics-sharelatex"); -rclient = require("redis-sharelatex").createClient(settings.redis.documentupdater) -Keys = settings.redis.documentupdater.key_schema +const rclient = require("redis-sharelatex").createClient(settings.redis.documentupdater); +const Keys = settings.redis.documentupdater.key_schema; -module.exports = DocumentUpdaterManager = - getDocument: (project_id, doc_id, fromVersion, callback = (error, exists, doclines, version) ->) -> - timer = new metrics.Timer("get-document") - url = "#{settings.apis.documentupdater.url}/project/#{project_id}/doc/#{doc_id}?fromVersion=#{fromVersion}" - logger.log {project_id, doc_id, fromVersion}, "getting doc from document updater" - request.get url, (err, res, body) -> - timer.done() - if err? - logger.error {err, url, project_id, doc_id}, "error getting doc from doc updater" - return callback(err) - if 200 <= res.statusCode < 300 - logger.log {project_id, doc_id}, "got doc from document document updater" - try - body = JSON.parse(body) - catch error - return callback(error) - callback null, body?.lines, body?.version, body?.ranges, body?.ops - else if res.statusCode in [404, 422] - err = new Error("doc updater could not load requested ops") - err.statusCode = res.statusCode - logger.warn {err, project_id, doc_id, url, fromVersion}, "doc updater could not load requested ops" - callback err - else - err = new Error("doc updater returned a non-success status code: #{res.statusCode}") - err.statusCode = res.statusCode - logger.error {err, project_id, doc_id, url}, "doc updater returned a non-success status code: #{res.statusCode}" - callback err +module.exports = (DocumentUpdaterManager = { + getDocument(project_id, doc_id, fromVersion, callback) { + if (callback == null) { callback = function(error, exists, doclines, version) {}; } + const timer = new metrics.Timer("get-document"); + const url = `${settings.apis.documentupdater.url}/project/${project_id}/doc/${doc_id}?fromVersion=${fromVersion}`; + logger.log({project_id, doc_id, fromVersion}, "getting doc from document updater"); + return request.get(url, function(err, res, body) { + timer.done(); + if (err != null) { + logger.error({err, url, project_id, doc_id}, "error getting doc from doc updater"); + return callback(err); + } + if (200 <= res.statusCode && res.statusCode < 300) { + logger.log({project_id, doc_id}, "got doc from document document updater"); + try { + body = JSON.parse(body); + } catch (error) { + return callback(error); + } + return callback(null, body != null ? body.lines : undefined, body != null ? body.version : undefined, body != null ? body.ranges : undefined, body != null ? body.ops : undefined); + } else if ([404, 422].includes(res.statusCode)) { + err = new Error("doc updater could not load requested ops"); + err.statusCode = res.statusCode; + logger.warn({err, project_id, doc_id, url, fromVersion}, "doc updater could not load requested ops"); + return callback(err); + } else { + err = new Error(`doc updater returned a non-success status code: ${res.statusCode}`); + err.statusCode = res.statusCode; + logger.error({err, project_id, doc_id, url}, `doc updater returned a non-success status code: ${res.statusCode}`); + return callback(err); + } + }); + }, - flushProjectToMongoAndDelete: (project_id, callback = ()->) -> - # this method is called when the last connected user leaves the project - logger.log project_id:project_id, "deleting project from document updater" - timer = new metrics.Timer("delete.mongo.project") - # flush the project in the background when all users have left - url = "#{settings.apis.documentupdater.url}/project/#{project_id}?background=true" + - (if settings.shutDownInProgress then "&shutdown=true" else "") - request.del url, (err, res, body)-> - timer.done() - if err? - logger.error {err, project_id}, "error deleting project from document updater" - return callback(err) - else if 200 <= res.statusCode < 300 - logger.log {project_id}, "deleted project from document updater" - return callback(null) - else - err = new Error("document updater returned a failure status code: #{res.statusCode}") - err.statusCode = res.statusCode - logger.error {err, project_id}, "document updater returned failure status code: #{res.statusCode}" - return callback(err) + flushProjectToMongoAndDelete(project_id, callback) { + // this method is called when the last connected user leaves the project + if (callback == null) { callback = function(){}; } + logger.log({project_id}, "deleting project from document updater"); + const timer = new metrics.Timer("delete.mongo.project"); + // flush the project in the background when all users have left + const url = `${settings.apis.documentupdater.url}/project/${project_id}?background=true` + + (settings.shutDownInProgress ? "&shutdown=true" : ""); + return request.del(url, function(err, res, body){ + timer.done(); + if (err != null) { + logger.error({err, project_id}, "error deleting project from document updater"); + return callback(err); + } else if (200 <= res.statusCode && res.statusCode < 300) { + logger.log({project_id}, "deleted project from document updater"); + return callback(null); + } else { + err = new Error(`document updater returned a failure status code: ${res.statusCode}`); + err.statusCode = res.statusCode; + logger.error({err, project_id}, `document updater returned failure status code: ${res.statusCode}`); + return callback(err); + } + }); + }, - queueChange: (project_id, doc_id, change, callback = ()->)-> - allowedKeys = [ 'doc', 'op', 'v', 'dupIfSource', 'meta', 'lastV', 'hash'] - change = _.pick change, allowedKeys - jsonChange = JSON.stringify change - if jsonChange.indexOf("\u0000") != -1 - # memory corruption check - error = new Error("null bytes found in op") - logger.error err: error, project_id: project_id, doc_id: doc_id, jsonChange: jsonChange, error.message - return callback(error) + queueChange(project_id, doc_id, change, callback){ + let error; + if (callback == null) { callback = function(){}; } + const allowedKeys = [ 'doc', 'op', 'v', 'dupIfSource', 'meta', 'lastV', 'hash']; + change = _.pick(change, allowedKeys); + const jsonChange = JSON.stringify(change); + if (jsonChange.indexOf("\u0000") !== -1) { + // memory corruption check + error = new Error("null bytes found in op"); + logger.error({err: error, project_id, doc_id, jsonChange}, error.message); + return callback(error); + } - updateSize = jsonChange.length - if updateSize > settings.maxUpdateSize - error = new Error("update is too large") - error.updateSize = updateSize - return callback(error) + const updateSize = jsonChange.length; + if (updateSize > settings.maxUpdateSize) { + error = new Error("update is too large"); + error.updateSize = updateSize; + return callback(error); + } - # record metric for each update added to queue - metrics.summary 'redis.pendingUpdates', updateSize, {status: 'push'} + // record metric for each update added to queue + metrics.summary('redis.pendingUpdates', updateSize, {status: 'push'}); - doc_key = "#{project_id}:#{doc_id}" - # Push onto pendingUpdates for doc_id first, because once the doc updater - # gets an entry on pending-updates-list, it starts processing. - rclient.rpush Keys.pendingUpdates({doc_id}), jsonChange, (error) -> - return callback(error) if error? - rclient.rpush "pending-updates-list", doc_key, callback + const doc_key = `${project_id}:${doc_id}`; + // Push onto pendingUpdates for doc_id first, because once the doc updater + // gets an entry on pending-updates-list, it starts processing. + return rclient.rpush(Keys.pendingUpdates({doc_id}), jsonChange, function(error) { + if (error != null) { return callback(error); } + return rclient.rpush("pending-updates-list", doc_key, callback); + }); + } +}); diff --git a/services/real-time/app/coffee/DrainManager.js b/services/real-time/app/coffee/DrainManager.js index 2590a96726..2f4067cc3c 100644 --- a/services/real-time/app/coffee/DrainManager.js +++ b/services/real-time/app/coffee/DrainManager.js @@ -1,39 +1,57 @@ -logger = require "logger-sharelatex" +/* + * decaffeinate suggestions: + * DS101: Remove unnecessary use of Array.from + * DS102: Remove unnecessary code created because of implicit returns + * Full docs: https://github.com/decaffeinate/decaffeinate/blob/master/docs/suggestions.md + */ +let DrainManager; +const logger = require("logger-sharelatex"); -module.exports = DrainManager = +module.exports = (DrainManager = { - startDrainTimeWindow: (io, minsToDrain)-> - drainPerMin = io.sockets.clients().length / minsToDrain - DrainManager.startDrain(io, Math.max(drainPerMin / 60, 4)) # enforce minimum drain rate + startDrainTimeWindow(io, minsToDrain){ + const drainPerMin = io.sockets.clients().length / minsToDrain; + return DrainManager.startDrain(io, Math.max(drainPerMin / 60, 4)); + }, // enforce minimum drain rate - startDrain: (io, rate) -> - # Clear out any old interval - clearInterval @interval - logger.log rate: rate, "starting drain" - if rate == 0 - return - else if rate < 1 - # allow lower drain rates - # e.g. rate=0.1 will drain one client every 10 seconds - pollingInterval = 1000 / rate - rate = 1 - else - pollingInterval = 1000 - @interval = setInterval () => - @reconnectNClients(io, rate) - , pollingInterval + startDrain(io, rate) { + // Clear out any old interval + let pollingInterval; + clearInterval(this.interval); + logger.log({rate}, "starting drain"); + if (rate === 0) { + return; + } else if (rate < 1) { + // allow lower drain rates + // e.g. rate=0.1 will drain one client every 10 seconds + pollingInterval = 1000 / rate; + rate = 1; + } else { + pollingInterval = 1000; + } + return this.interval = setInterval(() => { + return this.reconnectNClients(io, rate); + } + , pollingInterval); + }, - RECONNECTED_CLIENTS: {} - reconnectNClients: (io, N) -> - drainedCount = 0 - for client in io.sockets.clients() - if !@RECONNECTED_CLIENTS[client.id] - @RECONNECTED_CLIENTS[client.id] = true - logger.log {client_id: client.id}, "Asking client to reconnect gracefully" - client.emit "reconnectGracefully" - drainedCount++ - haveDrainedNClients = (drainedCount == N) - if haveDrainedNClients - break - if drainedCount < N - logger.log "All clients have been told to reconnectGracefully" + RECONNECTED_CLIENTS: {}, + reconnectNClients(io, N) { + let drainedCount = 0; + for (let client of Array.from(io.sockets.clients())) { + if (!this.RECONNECTED_CLIENTS[client.id]) { + this.RECONNECTED_CLIENTS[client.id] = true; + logger.log({client_id: client.id}, "Asking client to reconnect gracefully"); + client.emit("reconnectGracefully"); + drainedCount++; + } + const haveDrainedNClients = (drainedCount === N); + if (haveDrainedNClients) { + break; + } + } + if (drainedCount < N) { + return logger.log("All clients have been told to reconnectGracefully"); + } + } +}); diff --git a/services/real-time/app/coffee/Errors.js b/services/real-time/app/coffee/Errors.js index d6ef3fd71d..2ae4fbd6ab 100644 --- a/services/real-time/app/coffee/Errors.js +++ b/services/real-time/app/coffee/Errors.js @@ -1,10 +1,12 @@ -CodedError = (message, code) -> - error = new Error(message) - error.name = "CodedError" - error.code = code - error.__proto__ = CodedError.prototype - return error -CodedError.prototype.__proto__ = Error.prototype +let Errors; +var CodedError = function(message, code) { + const error = new Error(message); + error.name = "CodedError"; + error.code = code; + error.__proto__ = CodedError.prototype; + return error; +}; +CodedError.prototype.__proto__ = Error.prototype; -module.exports = Errors = - CodedError: CodedError +module.exports = (Errors = + {CodedError}); diff --git a/services/real-time/app/coffee/EventLogger.js b/services/real-time/app/coffee/EventLogger.js index 332973659b..bc01011687 100644 --- a/services/real-time/app/coffee/EventLogger.js +++ b/services/real-time/app/coffee/EventLogger.js @@ -1,60 +1,88 @@ -logger = require 'logger-sharelatex' -metrics = require 'metrics-sharelatex' -settings = require 'settings-sharelatex' +/* + * decaffeinate suggestions: + * DS102: Remove unnecessary code created because of implicit returns + * DS205: Consider reworking code to avoid use of IIFEs + * DS207: Consider shorter variations of null checks + * Full docs: https://github.com/decaffeinate/decaffeinate/blob/master/docs/suggestions.md + */ +let EventLogger; +const logger = require('logger-sharelatex'); +const metrics = require('metrics-sharelatex'); +const settings = require('settings-sharelatex'); -# keep track of message counters to detect duplicate and out of order events -# messsage ids have the format "UNIQUEHOSTKEY-COUNTER" +// keep track of message counters to detect duplicate and out of order events +// messsage ids have the format "UNIQUEHOSTKEY-COUNTER" -EVENT_LOG_COUNTER = {} -EVENT_LOG_TIMESTAMP = {} -EVENT_LAST_CLEAN_TIMESTAMP = 0 +const EVENT_LOG_COUNTER = {}; +const EVENT_LOG_TIMESTAMP = {}; +let EVENT_LAST_CLEAN_TIMESTAMP = 0; -# counter for debug logs -COUNTER = 0 +// counter for debug logs +let COUNTER = 0; -module.exports = EventLogger = +module.exports = (EventLogger = { - MAX_STALE_TIME_IN_MS: 3600 * 1000 + MAX_STALE_TIME_IN_MS: 3600 * 1000, - debugEvent: (channel, message) -> - if settings.debugEvents > 0 - logger.log {channel:channel, message:message, counter: COUNTER++}, "logging event" - settings.debugEvents-- + debugEvent(channel, message) { + if (settings.debugEvents > 0) { + logger.log({channel, message, counter: COUNTER++}, "logging event"); + return settings.debugEvents--; + } + }, - checkEventOrder: (channel, message_id, message) -> - return if typeof(message_id) isnt 'string' - return if !(result = message_id.match(/^(.*)-(\d+)$/)) - key = result[1] - count = parseInt(result[2], 0) - if !(count >= 0)# ignore checks if counter is not present - return - # store the last count in a hash for each host - previous = EventLogger._storeEventCount(key, count) - if !previous? || count == (previous + 1) - metrics.inc "event.#{channel}.valid", 0.001 # downsample high rate docupdater events - return # order is ok - if (count == previous) - metrics.inc "event.#{channel}.duplicate" - logger.warn {channel:channel, message_id:message_id}, "duplicate event" - return "duplicate" - else - metrics.inc "event.#{channel}.out-of-order" - logger.warn {channel:channel, message_id:message_id, key:key, previous: previous, count:count}, "out of order event" - return "out-of-order" + checkEventOrder(channel, message_id, message) { + let result; + if (typeof(message_id) !== 'string') { return; } + if (!(result = message_id.match(/^(.*)-(\d+)$/))) { return; } + const key = result[1]; + const count = parseInt(result[2], 0); + if (!(count >= 0)) {// ignore checks if counter is not present + return; + } + // store the last count in a hash for each host + const previous = EventLogger._storeEventCount(key, count); + if ((previous == null) || (count === (previous + 1))) { + metrics.inc(`event.${channel}.valid`, 0.001); // downsample high rate docupdater events + return; // order is ok + } + if (count === previous) { + metrics.inc(`event.${channel}.duplicate`); + logger.warn({channel, message_id}, "duplicate event"); + return "duplicate"; + } else { + metrics.inc(`event.${channel}.out-of-order`); + logger.warn({channel, message_id, key, previous, count}, "out of order event"); + return "out-of-order"; + } + }, - _storeEventCount: (key, count) -> - previous = EVENT_LOG_COUNTER[key] - now = Date.now() - EVENT_LOG_COUNTER[key] = count - EVENT_LOG_TIMESTAMP[key] = now - # periodically remove old counts - if (now - EVENT_LAST_CLEAN_TIMESTAMP) > EventLogger.MAX_STALE_TIME_IN_MS - EventLogger._cleanEventStream(now) - EVENT_LAST_CLEAN_TIMESTAMP = now - return previous + _storeEventCount(key, count) { + const previous = EVENT_LOG_COUNTER[key]; + const now = Date.now(); + EVENT_LOG_COUNTER[key] = count; + EVENT_LOG_TIMESTAMP[key] = now; + // periodically remove old counts + if ((now - EVENT_LAST_CLEAN_TIMESTAMP) > EventLogger.MAX_STALE_TIME_IN_MS) { + EventLogger._cleanEventStream(now); + EVENT_LAST_CLEAN_TIMESTAMP = now; + } + return previous; + }, - _cleanEventStream: (now) -> - for key, timestamp of EVENT_LOG_TIMESTAMP - if (now - timestamp) > EventLogger.MAX_STALE_TIME_IN_MS - delete EVENT_LOG_COUNTER[key] - delete EVENT_LOG_TIMESTAMP[key] \ No newline at end of file + _cleanEventStream(now) { + return (() => { + const result = []; + for (let key in EVENT_LOG_TIMESTAMP) { + const timestamp = EVENT_LOG_TIMESTAMP[key]; + if ((now - timestamp) > EventLogger.MAX_STALE_TIME_IN_MS) { + delete EVENT_LOG_COUNTER[key]; + result.push(delete EVENT_LOG_TIMESTAMP[key]); + } else { + result.push(undefined); + } + } + return result; + })(); + } +}); \ No newline at end of file diff --git a/services/real-time/app/coffee/HealthCheckManager.js b/services/real-time/app/coffee/HealthCheckManager.js index bcd3e2ed07..47da253993 100644 --- a/services/real-time/app/coffee/HealthCheckManager.js +++ b/services/real-time/app/coffee/HealthCheckManager.js @@ -1,52 +1,75 @@ -metrics = require "metrics-sharelatex" -logger = require("logger-sharelatex") +/* + * decaffeinate suggestions: + * DS102: Remove unnecessary code created because of implicit returns + * DS207: Consider shorter variations of null checks + * Full docs: https://github.com/decaffeinate/decaffeinate/blob/master/docs/suggestions.md + */ +let HealthCheckManager; +const metrics = require("metrics-sharelatex"); +const logger = require("logger-sharelatex"); -os = require "os" -HOST = os.hostname() -PID = process.pid -COUNT = 0 +const os = require("os"); +const HOST = os.hostname(); +const PID = process.pid; +let COUNT = 0; -CHANNEL_MANAGER = {} # hash of event checkers by channel name -CHANNEL_ERROR = {} # error status by channel name +const CHANNEL_MANAGER = {}; // hash of event checkers by channel name +const CHANNEL_ERROR = {}; // error status by channel name -module.exports = class HealthCheckManager - # create an instance of this class which checks that an event with a unique - # id is received only once within a timeout - constructor: (@channel, timeout = 1000) -> - # unique event string - @id = "host=#{HOST}:pid=#{PID}:count=#{COUNT++}" - # count of number of times the event is received - @count = 0 - # after a timeout check the status of the count - @handler = setTimeout () => - @setStatus() - , timeout - # use a timer to record the latency of the channel - @timer = new metrics.Timer("event.#{@channel}.latency") - # keep a record of these objects to dispatch on - CHANNEL_MANAGER[@channel] = @ - processEvent: (id) -> - # if this is our event record it - if id == @id - @count++ - @timer?.done() - @timer = null # only time the latency of the first event - setStatus: () -> - # if we saw the event anything other than a single time that is an error - if @count != 1 - logger.err channel:@channel, count:@count, id:@id, "redis channel health check error" - error = (@count != 1) - CHANNEL_ERROR[@channel] = error +module.exports = (HealthCheckManager = class HealthCheckManager { + // create an instance of this class which checks that an event with a unique + // id is received only once within a timeout + constructor(channel, timeout) { + // unique event string + this.channel = channel; + if (timeout == null) { timeout = 1000; } + this.id = `host=${HOST}:pid=${PID}:count=${COUNT++}`; + // count of number of times the event is received + this.count = 0; + // after a timeout check the status of the count + this.handler = setTimeout(() => { + return this.setStatus(); + } + , timeout); + // use a timer to record the latency of the channel + this.timer = new metrics.Timer(`event.${this.channel}.latency`); + // keep a record of these objects to dispatch on + CHANNEL_MANAGER[this.channel] = this; + } + processEvent(id) { + // if this is our event record it + if (id === this.id) { + this.count++; + if (this.timer != null) { + this.timer.done(); + } + return this.timer = null; // only time the latency of the first event + } + } + setStatus() { + // if we saw the event anything other than a single time that is an error + if (this.count !== 1) { + logger.err({channel:this.channel, count:this.count, id:this.id}, "redis channel health check error"); + } + const error = (this.count !== 1); + return CHANNEL_ERROR[this.channel] = error; + } - # class methods - @check: (channel, id) -> - # dispatch event to manager for channel - CHANNEL_MANAGER[channel]?.processEvent id - @status: () -> - # return status of all channels for logging - return CHANNEL_ERROR - @isFailing: () -> - # check if any channel status is bad - for channel, error of CHANNEL_ERROR - return true if error is true - return false + // class methods + static check(channel, id) { + // dispatch event to manager for channel + return (CHANNEL_MANAGER[channel] != null ? CHANNEL_MANAGER[channel].processEvent(id) : undefined); + } + static status() { + // return status of all channels for logging + return CHANNEL_ERROR; + } + static isFailing() { + // check if any channel status is bad + for (let channel in CHANNEL_ERROR) { + const error = CHANNEL_ERROR[channel]; + if (error === true) { return true; } + } + return false; + } +}); diff --git a/services/real-time/app/coffee/HttpApiController.js b/services/real-time/app/coffee/HttpApiController.js index 299d198f57..21a9f15628 100644 --- a/services/real-time/app/coffee/HttpApiController.js +++ b/services/real-time/app/coffee/HttpApiController.js @@ -1,35 +1,50 @@ -WebsocketLoadBalancer = require "./WebsocketLoadBalancer" -DrainManager = require "./DrainManager" -logger = require "logger-sharelatex" +/* + * decaffeinate suggestions: + * DS101: Remove unnecessary use of Array.from + * DS102: Remove unnecessary code created because of implicit returns + * Full docs: https://github.com/decaffeinate/decaffeinate/blob/master/docs/suggestions.md + */ +let HttpApiController; +const WebsocketLoadBalancer = require("./WebsocketLoadBalancer"); +const DrainManager = require("./DrainManager"); +const logger = require("logger-sharelatex"); -module.exports = HttpApiController = - sendMessage: (req, res, next) -> - logger.log {message: req.params.message}, "sending message" - if Array.isArray(req.body) - for payload in req.body - WebsocketLoadBalancer.emitToRoom req.params.project_id, req.params.message, payload - else - WebsocketLoadBalancer.emitToRoom req.params.project_id, req.params.message, req.body - res.send 204 # No content +module.exports = (HttpApiController = { + sendMessage(req, res, next) { + logger.log({message: req.params.message}, "sending message"); + if (Array.isArray(req.body)) { + for (let payload of Array.from(req.body)) { + WebsocketLoadBalancer.emitToRoom(req.params.project_id, req.params.message, payload); + } + } else { + WebsocketLoadBalancer.emitToRoom(req.params.project_id, req.params.message, req.body); + } + return res.send(204); + }, // No content - startDrain: (req, res, next) -> - io = req.app.get("io") - rate = req.query.rate or "4" - rate = parseFloat(rate) || 0 - logger.log {rate}, "setting client drain rate" - DrainManager.startDrain io, rate - res.send 204 + startDrain(req, res, next) { + const io = req.app.get("io"); + let rate = req.query.rate || "4"; + rate = parseFloat(rate) || 0; + logger.log({rate}, "setting client drain rate"); + DrainManager.startDrain(io, rate); + return res.send(204); + }, - disconnectClient: (req, res, next) -> - io = req.app.get("io") - client_id = req.params.client_id - client = io.sockets.sockets[client_id] + disconnectClient(req, res, next) { + const io = req.app.get("io"); + const { + client_id + } = req.params; + const client = io.sockets.sockets[client_id]; - if !client - logger.info({client_id}, "api: client already disconnected") - res.sendStatus(404) - return - logger.warn({client_id}, "api: requesting client disconnect") - client.on "disconnect", () -> - res.sendStatus(204) - client.disconnect() + if (!client) { + logger.info({client_id}, "api: client already disconnected"); + res.sendStatus(404); + return; + } + logger.warn({client_id}, "api: requesting client disconnect"); + client.on("disconnect", () => res.sendStatus(204)); + return client.disconnect(); + } +}); diff --git a/services/real-time/app/coffee/HttpController.js b/services/real-time/app/coffee/HttpController.js index 1fc74e8c16..aa17c6f6d8 100644 --- a/services/real-time/app/coffee/HttpController.js +++ b/services/real-time/app/coffee/HttpController.js @@ -1,35 +1,53 @@ -async = require "async" +/* + * decaffeinate suggestions: + * DS102: Remove unnecessary code created because of implicit returns + * DS207: Consider shorter variations of null checks + * Full docs: https://github.com/decaffeinate/decaffeinate/blob/master/docs/suggestions.md + */ +let HttpController; +const async = require("async"); -module.exports = HttpController = - # The code in this controller is hard to unit test because of a lot of - # dependencies on internal socket.io methods. It is not critical to the running - # of ShareLaTeX, and is only used for getting stats about connected clients, - # and for checking internal state in acceptance tests. The acceptances tests - # should provide appropriate coverage. - _getConnectedClientView: (ioClient, callback = (error, client) ->) -> - client_id = ioClient.id - {project_id, user_id, first_name, last_name, email, connected_time} = ioClient.ol_context - client = {client_id, project_id, user_id, first_name, last_name, email, connected_time} - client.rooms = [] - for name, joined of ioClient.manager.roomClients[client_id] - if joined and name != "" - client.rooms.push name.replace(/^\//, "") # Remove leading / - callback(null, client) +module.exports = (HttpController = { + // The code in this controller is hard to unit test because of a lot of + // dependencies on internal socket.io methods. It is not critical to the running + // of ShareLaTeX, and is only used for getting stats about connected clients, + // and for checking internal state in acceptance tests. The acceptances tests + // should provide appropriate coverage. + _getConnectedClientView(ioClient, callback) { + if (callback == null) { callback = function(error, client) {}; } + const client_id = ioClient.id; + const {project_id, user_id, first_name, last_name, email, connected_time} = ioClient.ol_context; + const client = {client_id, project_id, user_id, first_name, last_name, email, connected_time}; + client.rooms = []; + for (let name in ioClient.manager.roomClients[client_id]) { + const joined = ioClient.manager.roomClients[client_id][name]; + if (joined && (name !== "")) { + client.rooms.push(name.replace(/^\//, "")); // Remove leading / + } + } + return callback(null, client); + }, - getConnectedClients: (req, res, next) -> - io = req.app.get("io") - ioClients = io.sockets.clients() - async.map ioClients, HttpController._getConnectedClientView, (error, clients) -> - return next(error) if error? - res.json clients + getConnectedClients(req, res, next) { + const io = req.app.get("io"); + const ioClients = io.sockets.clients(); + return async.map(ioClients, HttpController._getConnectedClientView, function(error, clients) { + if (error != null) { return next(error); } + return res.json(clients); + }); + }, - getConnectedClient: (req, res, next) -> - {client_id} = req.params - io = req.app.get("io") - ioClient = io.sockets.sockets[client_id] - if !ioClient - res.sendStatus(404) - return - HttpController._getConnectedClientView ioClient, (error, client) -> - return next(error) if error? - res.json client + getConnectedClient(req, res, next) { + const {client_id} = req.params; + const io = req.app.get("io"); + const ioClient = io.sockets.sockets[client_id]; + if (!ioClient) { + res.sendStatus(404); + return; + } + return HttpController._getConnectedClientView(ioClient, function(error, client) { + if (error != null) { return next(error); } + return res.json(client); + }); + } +}); diff --git a/services/real-time/app/coffee/RedisClientManager.js b/services/real-time/app/coffee/RedisClientManager.js index 1d573df9b8..7bd33ca914 100644 --- a/services/real-time/app/coffee/RedisClientManager.js +++ b/services/real-time/app/coffee/RedisClientManager.js @@ -1,18 +1,35 @@ -redis = require("redis-sharelatex") -logger = require 'logger-sharelatex' +/* + * decaffeinate suggestions: + * DS101: Remove unnecessary use of Array.from + * DS205: Consider reworking code to avoid use of IIFEs + * DS207: Consider shorter variations of null checks + * Full docs: https://github.com/decaffeinate/decaffeinate/blob/master/docs/suggestions.md + */ +let RedisClientManager; +const redis = require("redis-sharelatex"); +const logger = require('logger-sharelatex'); -module.exports = RedisClientManager = - createClientList: (configs...) -> - # create a dynamic list of redis clients, excluding any configurations which are not defined - clientList = for x in configs when x? - redisType = if x.cluster? - "cluster" - else if x.sentinels? - "sentinel" - else if x.host? - "single" - else - "unknown" - logger.log {redis: redisType}, "creating redis client" - redis.createClient(x) - return clientList \ No newline at end of file +module.exports = (RedisClientManager = { + createClientList(...configs) { + // create a dynamic list of redis clients, excluding any configurations which are not defined + const clientList = (() => { + const result = []; + for (let x of Array.from(configs)) { + if (x != null) { + const redisType = (x.cluster != null) ? + "cluster" + : (x.sentinels != null) ? + "sentinel" + : (x.host != null) ? + "single" + : + "unknown"; + logger.log({redis: redisType}, "creating redis client"); + result.push(redis.createClient(x)); + } + } + return result; + })(); + return clientList; + } +}); \ No newline at end of file diff --git a/services/real-time/app/coffee/RoomManager.js b/services/real-time/app/coffee/RoomManager.js index 25684ed558..c7047e90c0 100644 --- a/services/real-time/app/coffee/RoomManager.js +++ b/services/real-time/app/coffee/RoomManager.js @@ -1,110 +1,154 @@ -logger = require 'logger-sharelatex' -metrics = require "metrics-sharelatex" -{EventEmitter} = require 'events' +/* + * decaffeinate suggestions: + * DS101: Remove unnecessary use of Array.from + * DS102: Remove unnecessary code created because of implicit returns + * DS103: Rewrite code to no longer use __guard__ + * DS205: Consider reworking code to avoid use of IIFEs + * DS207: Consider shorter variations of null checks + * Full docs: https://github.com/decaffeinate/decaffeinate/blob/master/docs/suggestions.md + */ +let RoomManager; +const logger = require('logger-sharelatex'); +const metrics = require("metrics-sharelatex"); +const {EventEmitter} = require('events'); -IdMap = new Map() # keep track of whether ids are from projects or docs -RoomEvents = new EventEmitter() # emits {project,doc}-active and {project,doc}-empty events +const IdMap = new Map(); // keep track of whether ids are from projects or docs +const RoomEvents = new EventEmitter(); // emits {project,doc}-active and {project,doc}-empty events -# Manage socket.io rooms for individual projects and docs -# -# The first time someone joins a project or doc we emit a 'project-active' or -# 'doc-active' event. -# -# When the last person leaves a project or doc, we emit 'project-empty' or -# 'doc-empty' event. -# -# The pubsub side is handled by ChannelManager +// Manage socket.io rooms for individual projects and docs +// +// The first time someone joins a project or doc we emit a 'project-active' or +// 'doc-active' event. +// +// When the last person leaves a project or doc, we emit 'project-empty' or +// 'doc-empty' event. +// +// The pubsub side is handled by ChannelManager -module.exports = RoomManager = +module.exports = (RoomManager = { - joinProject: (client, project_id, callback = () ->) -> - @joinEntity client, "project", project_id, callback + joinProject(client, project_id, callback) { + if (callback == null) { callback = function() {}; } + return this.joinEntity(client, "project", project_id, callback); + }, - joinDoc: (client, doc_id, callback = () ->) -> - @joinEntity client, "doc", doc_id, callback + joinDoc(client, doc_id, callback) { + if (callback == null) { callback = function() {}; } + return this.joinEntity(client, "doc", doc_id, callback); + }, - leaveDoc: (client, doc_id) -> - @leaveEntity client, "doc", doc_id + leaveDoc(client, doc_id) { + return this.leaveEntity(client, "doc", doc_id); + }, - leaveProjectAndDocs: (client) -> - # what rooms is this client in? we need to leave them all. socket.io - # will cause us to leave the rooms, so we only need to manage our - # channel subscriptions... but it will be safer if we leave them - # explicitly, and then socket.io will just regard this as a client that - # has not joined any rooms and do a final disconnection. - roomsToLeave = @_roomsClientIsIn(client) - logger.log {client: client.id, roomsToLeave: roomsToLeave}, "client leaving project" - for id in roomsToLeave - entity = IdMap.get(id) - @leaveEntity client, entity, id + leaveProjectAndDocs(client) { + // what rooms is this client in? we need to leave them all. socket.io + // will cause us to leave the rooms, so we only need to manage our + // channel subscriptions... but it will be safer if we leave them + // explicitly, and then socket.io will just regard this as a client that + // has not joined any rooms and do a final disconnection. + const roomsToLeave = this._roomsClientIsIn(client); + logger.log({client: client.id, roomsToLeave}, "client leaving project"); + return (() => { + const result = []; + for (let id of Array.from(roomsToLeave)) { + const entity = IdMap.get(id); + result.push(this.leaveEntity(client, entity, id)); + } + return result; + })(); + }, - emitOnCompletion: (promiseList, eventName) -> - Promise.all(promiseList) - .then(() -> RoomEvents.emit(eventName)) - .catch((err) -> RoomEvents.emit(eventName, err)) + emitOnCompletion(promiseList, eventName) { + return Promise.all(promiseList) + .then(() => RoomEvents.emit(eventName)) + .catch(err => RoomEvents.emit(eventName, err)); + }, - eventSource: () -> - return RoomEvents + eventSource() { + return RoomEvents; + }, - joinEntity: (client, entity, id, callback) -> - beforeCount = @_clientsInRoom(client, id) - # client joins room immediately but joinDoc request does not complete - # until room is subscribed - client.join id - # is this a new room? if so, subscribe - if beforeCount == 0 - logger.log {entity, id}, "room is now active" - RoomEvents.once "#{entity}-subscribed-#{id}", (err) -> - # only allow the client to join when all the relevant channels have subscribed - logger.log {client: client.id, entity, id, beforeCount}, "client joined new room and subscribed to channel" - callback(err) - RoomEvents.emit "#{entity}-active", id - IdMap.set(id, entity) - # keep track of the number of listeners - metrics.gauge "room-listeners", RoomEvents.eventNames().length - else - logger.log {client: client.id, entity, id, beforeCount}, "client joined existing room" - client.join id - callback() + joinEntity(client, entity, id, callback) { + const beforeCount = this._clientsInRoom(client, id); + // client joins room immediately but joinDoc request does not complete + // until room is subscribed + client.join(id); + // is this a new room? if so, subscribe + if (beforeCount === 0) { + logger.log({entity, id}, "room is now active"); + RoomEvents.once(`${entity}-subscribed-${id}`, function(err) { + // only allow the client to join when all the relevant channels have subscribed + logger.log({client: client.id, entity, id, beforeCount}, "client joined new room and subscribed to channel"); + return callback(err); + }); + RoomEvents.emit(`${entity}-active`, id); + IdMap.set(id, entity); + // keep track of the number of listeners + return metrics.gauge("room-listeners", RoomEvents.eventNames().length); + } else { + logger.log({client: client.id, entity, id, beforeCount}, "client joined existing room"); + client.join(id); + return callback(); + } + }, - leaveEntity: (client, entity, id) -> - # Ignore any requests to leave when the client is not actually in the - # room. This can happen if the client sends spurious leaveDoc requests - # for old docs after a reconnection. - # This can now happen all the time, as we skip the join for clients that - # disconnect before joinProject/joinDoc completed. - if !@_clientAlreadyInRoom(client, id) - logger.log {client: client.id, entity, id}, "ignoring request from client to leave room it is not in" - return - client.leave id - afterCount = @_clientsInRoom(client, id) - logger.log {client: client.id, entity, id, afterCount}, "client left room" - # is the room now empty? if so, unsubscribe - if !entity? - logger.error {entity: id}, "unknown entity when leaving with id" - return - if afterCount == 0 - logger.log {entity, id}, "room is now empty" - RoomEvents.emit "#{entity}-empty", id - IdMap.delete(id) - metrics.gauge "room-listeners", RoomEvents.eventNames().length + leaveEntity(client, entity, id) { + // Ignore any requests to leave when the client is not actually in the + // room. This can happen if the client sends spurious leaveDoc requests + // for old docs after a reconnection. + // This can now happen all the time, as we skip the join for clients that + // disconnect before joinProject/joinDoc completed. + if (!this._clientAlreadyInRoom(client, id)) { + logger.log({client: client.id, entity, id}, "ignoring request from client to leave room it is not in"); + return; + } + client.leave(id); + const afterCount = this._clientsInRoom(client, id); + logger.log({client: client.id, entity, id, afterCount}, "client left room"); + // is the room now empty? if so, unsubscribe + if ((entity == null)) { + logger.error({entity: id}, "unknown entity when leaving with id"); + return; + } + if (afterCount === 0) { + logger.log({entity, id}, "room is now empty"); + RoomEvents.emit(`${entity}-empty`, id); + IdMap.delete(id); + return metrics.gauge("room-listeners", RoomEvents.eventNames().length); + } + }, - # internal functions below, these access socket.io rooms data directly and - # will need updating for socket.io v2 + // internal functions below, these access socket.io rooms data directly and + // will need updating for socket.io v2 - _clientsInRoom: (client, room) -> - nsp = client.namespace.name - name = (nsp + '/') + room; - return (client.manager?.rooms?[name] || []).length + _clientsInRoom(client, room) { + const nsp = client.namespace.name; + const name = (nsp + '/') + room; + return (__guard__(client.manager != null ? client.manager.rooms : undefined, x => x[name]) || []).length; + }, - _roomsClientIsIn: (client) -> - roomList = for fullRoomPath of client.manager.roomClients?[client.id] when fullRoomPath isnt '' - # strip socket.io prefix from room to get original id - [prefix, room] = fullRoomPath.split('/', 2) - room - return roomList + _roomsClientIsIn(client) { + const roomList = (() => { + const result = []; + for (let fullRoomPath in (client.manager.roomClients != null ? client.manager.roomClients[client.id] : undefined)) { + // strip socket.io prefix from room to get original id + if (fullRoomPath !== '') { + const [prefix, room] = Array.from(fullRoomPath.split('/', 2)); + result.push(room); + } + } + return result; + })(); + return roomList; + }, - _clientAlreadyInRoom: (client, room) -> - nsp = client.namespace.name - name = (nsp + '/') + room; - return client.manager.roomClients?[client.id]?[name] \ No newline at end of file + _clientAlreadyInRoom(client, room) { + const nsp = client.namespace.name; + const name = (nsp + '/') + room; + return __guard__(client.manager.roomClients != null ? client.manager.roomClients[client.id] : undefined, x => x[name]); + } +}); +function __guard__(value, transform) { + return (typeof value !== 'undefined' && value !== null) ? transform(value) : undefined; +} \ No newline at end of file diff --git a/services/real-time/app/coffee/Router.js b/services/real-time/app/coffee/Router.js index 3d891f1476..c7ea84192b 100644 --- a/services/real-time/app/coffee/Router.js +++ b/services/real-time/app/coffee/Router.js @@ -1,188 +1,264 @@ -metrics = require "metrics-sharelatex" -logger = require "logger-sharelatex" -settings = require "settings-sharelatex" -WebsocketController = require "./WebsocketController" -HttpController = require "./HttpController" -HttpApiController = require "./HttpApiController" -bodyParser = require "body-parser" -base64id = require("base64id") +/* + * decaffeinate suggestions: + * DS101: Remove unnecessary use of Array.from + * DS102: Remove unnecessary code created because of implicit returns + * DS103: Rewrite code to no longer use __guard__ + * DS207: Consider shorter variations of null checks + * Full docs: https://github.com/decaffeinate/decaffeinate/blob/master/docs/suggestions.md + */ +let Router; +const metrics = require("metrics-sharelatex"); +const logger = require("logger-sharelatex"); +const settings = require("settings-sharelatex"); +const WebsocketController = require("./WebsocketController"); +const HttpController = require("./HttpController"); +const HttpApiController = require("./HttpApiController"); +const bodyParser = require("body-parser"); +const base64id = require("base64id"); -basicAuth = require('basic-auth-connect') -httpAuth = basicAuth (user, pass)-> - isValid = user == settings.internal.realTime.user and pass == settings.internal.realTime.pass - if !isValid - logger.err user:user, pass:pass, "invalid login details" - return isValid +const basicAuth = require('basic-auth-connect'); +const httpAuth = basicAuth(function(user, pass){ + const isValid = (user === settings.internal.realTime.user) && (pass === settings.internal.realTime.pass); + if (!isValid) { + logger.err({user, pass}, "invalid login details"); + } + return isValid; +}); -module.exports = Router = - _handleError: (callback = ((error) ->), error, client, method, attrs = {}) -> - for key in ["project_id", "doc_id", "user_id"] - attrs[key] = client.ol_context[key] - attrs.client_id = client.id - attrs.err = error - if error.name == "CodedError" - logger.warn attrs, error.message, code: error.code - return callback {message: error.message, code: error.code} - if error.message == 'unexpected arguments' - # the payload might be very large, put it on level info - logger.log attrs, 'unexpected arguments' - metrics.inc 'unexpected-arguments', 1, { status: method } - return callback { message: error.message } - if error.message in ["not authorized", "doc updater could not load requested ops", "no project_id found on client"] - logger.warn attrs, error.message - return callback {message: error.message} - else - logger.error attrs, "server side error in #{method}" - # Don't return raw error to prevent leaking server side info - return callback {message: "Something went wrong in real-time service"} +module.exports = (Router = { + _handleError(callback, error, client, method, attrs) { + if (callback == null) { callback = function(error) {}; } + if (attrs == null) { attrs = {}; } + for (let key of ["project_id", "doc_id", "user_id"]) { + attrs[key] = client.ol_context[key]; + } + attrs.client_id = client.id; + attrs.err = error; + if (error.name === "CodedError") { + logger.warn(attrs, error.message, {code: error.code}); + return callback({message: error.message, code: error.code}); + } + if (error.message === 'unexpected arguments') { + // the payload might be very large, put it on level info + logger.log(attrs, 'unexpected arguments'); + metrics.inc('unexpected-arguments', 1, { status: method }); + return callback({ message: error.message }); + } + if (["not authorized", "doc updater could not load requested ops", "no project_id found on client"].includes(error.message)) { + logger.warn(attrs, error.message); + return callback({message: error.message}); + } else { + logger.error(attrs, `server side error in ${method}`); + // Don't return raw error to prevent leaking server side info + return callback({message: "Something went wrong in real-time service"}); + } + }, - _handleInvalidArguments: (client, method, args) -> - error = new Error("unexpected arguments") - callback = args[args.length - 1] - if typeof callback != 'function' - callback = (() ->) - attrs = {arguments: args} - Router._handleError(callback, error, client, method, attrs) + _handleInvalidArguments(client, method, args) { + const error = new Error("unexpected arguments"); + let callback = args[args.length - 1]; + if (typeof callback !== 'function') { + callback = (function() {}); + } + const attrs = {arguments: args}; + return Router._handleError(callback, error, client, method, attrs); + }, - configure: (app, io, session) -> - app.set("io", io) - app.get "/clients", HttpController.getConnectedClients - app.get "/clients/:client_id", HttpController.getConnectedClient + configure(app, io, session) { + app.set("io", io); + app.get("/clients", HttpController.getConnectedClients); + app.get("/clients/:client_id", HttpController.getConnectedClient); - app.post "/project/:project_id/message/:message", httpAuth, bodyParser.json(limit: "5mb"), HttpApiController.sendMessage + app.post("/project/:project_id/message/:message", httpAuth, bodyParser.json({limit: "5mb"}), HttpApiController.sendMessage); - app.post "/drain", httpAuth, HttpApiController.startDrain - app.post "/client/:client_id/disconnect", httpAuth, HttpApiController.disconnectClient + app.post("/drain", httpAuth, HttpApiController.startDrain); + app.post("/client/:client_id/disconnect", httpAuth, HttpApiController.disconnectClient); - session.on 'connection', (error, client, session) -> - # init client context, we may access it in Router._handleError before - # setting any values - client.ol_context = {} + return session.on('connection', function(error, client, session) { + // init client context, we may access it in Router._handleError before + // setting any values + let user; + client.ol_context = {}; - client?.on "error", (err) -> - logger.err { clientErr: err }, "socket.io client error" - if client.connected - client.emit("reconnectGracefully") - client.disconnect() + if (client != null) { + client.on("error", function(err) { + logger.err({ clientErr: err }, "socket.io client error"); + if (client.connected) { + client.emit("reconnectGracefully"); + return client.disconnect(); + } + }); + } - if settings.shutDownInProgress - client.emit("connectionRejected", {message: "retry"}) - client.disconnect() - return + if (settings.shutDownInProgress) { + client.emit("connectionRejected", {message: "retry"}); + client.disconnect(); + return; + } - if client? and error?.message?.match(/could not look up session by key/) - logger.warn err: error, client: client?, session: session?, "invalid session" - # tell the client to reauthenticate if it has an invalid session key - client.emit("connectionRejected", {message: "invalid session"}) - client.disconnect() - return + if ((client != null) && __guard__(error != null ? error.message : undefined, x => x.match(/could not look up session by key/))) { + logger.warn({err: error, client: (client != null), session: (session != null)}, "invalid session"); + // tell the client to reauthenticate if it has an invalid session key + client.emit("connectionRejected", {message: "invalid session"}); + client.disconnect(); + return; + } - if error? - logger.err err: error, client: client?, session: session?, "error when client connected" - client?.emit("connectionRejected", {message: "error"}) - client?.disconnect() - return + if (error != null) { + logger.err({err: error, client: (client != null), session: (session != null)}, "error when client connected"); + if (client != null) { + client.emit("connectionRejected", {message: "error"}); + } + if (client != null) { + client.disconnect(); + } + return; + } - # send positive confirmation that the client has a valid connection - client.publicId = 'P.' + base64id.generateId() - client.emit("connectionAccepted", null, client.publicId) + // send positive confirmation that the client has a valid connection + client.publicId = 'P.' + base64id.generateId(); + client.emit("connectionAccepted", null, client.publicId); - metrics.inc('socket-io.connection') - metrics.gauge('socket-io.clients', io.sockets.clients()?.length) + metrics.inc('socket-io.connection'); + metrics.gauge('socket-io.clients', __guard__(io.sockets.clients(), x1 => x1.length)); - logger.log session: session, client_id: client.id, "client connected" + logger.log({session, client_id: client.id}, "client connected"); - if session?.passport?.user? - user = session.passport.user - else if session?.user? - user = session.user - else - user = {_id: "anonymous-user"} + if (__guard__(session != null ? session.passport : undefined, x2 => x2.user) != null) { + ({ + user + } = session.passport); + } else if ((session != null ? session.user : undefined) != null) { + ({ + user + } = session); + } else { + user = {_id: "anonymous-user"}; + } - client.on "joinProject", (data = {}, callback) -> - if typeof callback != 'function' - return Router._handleInvalidArguments(client, 'joinProject', arguments) + client.on("joinProject", function(data, callback) { + if (data == null) { data = {}; } + if (typeof callback !== 'function') { + return Router._handleInvalidArguments(client, 'joinProject', arguments); + } - if data.anonymousAccessToken - user.anonymousAccessToken = data.anonymousAccessToken - WebsocketController.joinProject client, user, data.project_id, (err, args...) -> - if err? - Router._handleError callback, err, client, "joinProject", {project_id: data.project_id, user_id: user?.id} - else - callback(null, args...) + if (data.anonymousAccessToken) { + user.anonymousAccessToken = data.anonymousAccessToken; + } + return WebsocketController.joinProject(client, user, data.project_id, function(err, ...args) { + if (err != null) { + return Router._handleError(callback, err, client, "joinProject", {project_id: data.project_id, user_id: (user != null ? user.id : undefined)}); + } else { + return callback(null, ...Array.from(args)); + } + }); + }); - client.on "disconnect", () -> - metrics.inc('socket-io.disconnect') - metrics.gauge('socket-io.clients', io.sockets.clients()?.length - 1) + client.on("disconnect", function() { + metrics.inc('socket-io.disconnect'); + metrics.gauge('socket-io.clients', __guard__(io.sockets.clients(), x3 => x3.length) - 1); - WebsocketController.leaveProject io, client, (err) -> - if err? - Router._handleError (() ->), err, client, "leaveProject" + return WebsocketController.leaveProject(io, client, function(err) { + if (err != null) { + return Router._handleError((function() {}), err, client, "leaveProject"); + } + }); + }); - # Variadic. The possible arguments: - # doc_id, callback - # doc_id, fromVersion, callback - # doc_id, options, callback - # doc_id, fromVersion, options, callback - client.on "joinDoc", (doc_id, fromVersion, options, callback) -> - if typeof fromVersion == "function" and !options - callback = fromVersion - fromVersion = -1 - options = {} - else if typeof fromVersion == "number" and typeof options == "function" - callback = options - options = {} - else if typeof fromVersion == "object" and typeof options == "function" - callback = options - options = fromVersion - fromVersion = -1 - else if typeof fromVersion == "number" and typeof options == "object" and typeof callback == 'function' - # Called with 4 args, things are as expected - else - return Router._handleInvalidArguments(client, 'joinDoc', arguments) + // Variadic. The possible arguments: + // doc_id, callback + // doc_id, fromVersion, callback + // doc_id, options, callback + // doc_id, fromVersion, options, callback + client.on("joinDoc", function(doc_id, fromVersion, options, callback) { + if ((typeof fromVersion === "function") && !options) { + callback = fromVersion; + fromVersion = -1; + options = {}; + } else if ((typeof fromVersion === "number") && (typeof options === "function")) { + callback = options; + options = {}; + } else if ((typeof fromVersion === "object") && (typeof options === "function")) { + callback = options; + options = fromVersion; + fromVersion = -1; + } else if ((typeof fromVersion === "number") && (typeof options === "object") && (typeof callback === 'function')) { + // Called with 4 args, things are as expected + } else { + return Router._handleInvalidArguments(client, 'joinDoc', arguments); + } - WebsocketController.joinDoc client, doc_id, fromVersion, options, (err, args...) -> - if err? - Router._handleError callback, err, client, "joinDoc", {doc_id, fromVersion} - else - callback(null, args...) + return WebsocketController.joinDoc(client, doc_id, fromVersion, options, function(err, ...args) { + if (err != null) { + return Router._handleError(callback, err, client, "joinDoc", {doc_id, fromVersion}); + } else { + return callback(null, ...Array.from(args)); + } + }); + }); - client.on "leaveDoc", (doc_id, callback) -> - if typeof callback != 'function' - return Router._handleInvalidArguments(client, 'leaveDoc', arguments) + client.on("leaveDoc", function(doc_id, callback) { + if (typeof callback !== 'function') { + return Router._handleInvalidArguments(client, 'leaveDoc', arguments); + } - WebsocketController.leaveDoc client, doc_id, (err, args...) -> - if err? - Router._handleError callback, err, client, "leaveDoc" - else - callback(null, args...) + return WebsocketController.leaveDoc(client, doc_id, function(err, ...args) { + if (err != null) { + return Router._handleError(callback, err, client, "leaveDoc"); + } else { + return callback(null, ...Array.from(args)); + } + }); + }); - client.on "clientTracking.getConnectedUsers", (callback = (error, users) ->) -> - if typeof callback != 'function' - return Router._handleInvalidArguments(client, 'clientTracking.getConnectedUsers', arguments) + client.on("clientTracking.getConnectedUsers", function(callback) { + if (callback == null) { callback = function(error, users) {}; } + if (typeof callback !== 'function') { + return Router._handleInvalidArguments(client, 'clientTracking.getConnectedUsers', arguments); + } - WebsocketController.getConnectedUsers client, (err, users) -> - if err? - Router._handleError callback, err, client, "clientTracking.getConnectedUsers" - else - callback(null, users) + return WebsocketController.getConnectedUsers(client, function(err, users) { + if (err != null) { + return Router._handleError(callback, err, client, "clientTracking.getConnectedUsers"); + } else { + return callback(null, users); + } + }); + }); - client.on "clientTracking.updatePosition", (cursorData, callback = (error) ->) -> - if typeof callback != 'function' - return Router._handleInvalidArguments(client, 'clientTracking.updatePosition', arguments) + client.on("clientTracking.updatePosition", function(cursorData, callback) { + if (callback == null) { callback = function(error) {}; } + if (typeof callback !== 'function') { + return Router._handleInvalidArguments(client, 'clientTracking.updatePosition', arguments); + } - WebsocketController.updateClientPosition client, cursorData, (err) -> - if err? - Router._handleError callback, err, client, "clientTracking.updatePosition" - else - callback() + return WebsocketController.updateClientPosition(client, cursorData, function(err) { + if (err != null) { + return Router._handleError(callback, err, client, "clientTracking.updatePosition"); + } else { + return callback(); + } + }); + }); - client.on "applyOtUpdate", (doc_id, update, callback = (error) ->) -> - if typeof callback != 'function' - return Router._handleInvalidArguments(client, 'applyOtUpdate', arguments) + return client.on("applyOtUpdate", function(doc_id, update, callback) { + if (callback == null) { callback = function(error) {}; } + if (typeof callback !== 'function') { + return Router._handleInvalidArguments(client, 'applyOtUpdate', arguments); + } - WebsocketController.applyOtUpdate client, doc_id, update, (err) -> - if err? - Router._handleError callback, err, client, "applyOtUpdate", {doc_id, update} - else - callback() + return WebsocketController.applyOtUpdate(client, doc_id, update, function(err) { + if (err != null) { + return Router._handleError(callback, err, client, "applyOtUpdate", {doc_id, update}); + } else { + return callback(); + } + }); + }); + }); + } +}); + +function __guard__(value, transform) { + return (typeof value !== 'undefined' && value !== null) ? transform(value) : undefined; +} \ No newline at end of file diff --git a/services/real-time/app/coffee/SafeJsonParse.js b/services/real-time/app/coffee/SafeJsonParse.js index afeb72f96e..4c058053b7 100644 --- a/services/real-time/app/coffee/SafeJsonParse.js +++ b/services/real-time/app/coffee/SafeJsonParse.js @@ -1,13 +1,25 @@ -Settings = require "settings-sharelatex" -logger = require "logger-sharelatex" +/* + * decaffeinate suggestions: + * DS102: Remove unnecessary code created because of implicit returns + * DS207: Consider shorter variations of null checks + * Full docs: https://github.com/decaffeinate/decaffeinate/blob/master/docs/suggestions.md + */ +const Settings = require("settings-sharelatex"); +const logger = require("logger-sharelatex"); -module.exports = - parse: (data, callback = (error, parsed) ->) -> - if data.length > Settings.maxUpdateSize - logger.error {head: data.slice(0,1024), length: data.length}, "data too large to parse" - return callback new Error("data too large to parse") - try - parsed = JSON.parse(data) - catch e - return callback e - callback null, parsed \ No newline at end of file +module.exports = { + parse(data, callback) { + let parsed; + if (callback == null) { callback = function(error, parsed) {}; } + if (data.length > Settings.maxUpdateSize) { + logger.error({head: data.slice(0,1024), length: data.length}, "data too large to parse"); + return callback(new Error("data too large to parse")); + } + try { + parsed = JSON.parse(data); + } catch (e) { + return callback(e); + } + return callback(null, parsed); + } +}; \ No newline at end of file diff --git a/services/real-time/app/coffee/SessionSockets.js b/services/real-time/app/coffee/SessionSockets.js index 229e07b3bb..533fed3d4c 100644 --- a/services/real-time/app/coffee/SessionSockets.js +++ b/services/real-time/app/coffee/SessionSockets.js @@ -1,23 +1,34 @@ -{EventEmitter} = require('events') +/* + * decaffeinate suggestions: + * DS102: Remove unnecessary code created because of implicit returns + * Full docs: https://github.com/decaffeinate/decaffeinate/blob/master/docs/suggestions.md + */ +const {EventEmitter} = require('events'); -module.exports = (io, sessionStore, cookieParser, cookieName) -> - missingSessionError = new Error('could not look up session by key') +module.exports = function(io, sessionStore, cookieParser, cookieName) { + const missingSessionError = new Error('could not look up session by key'); - sessionSockets = new EventEmitter() - next = (error, socket, session) -> - sessionSockets.emit 'connection', error, socket, session + const sessionSockets = new EventEmitter(); + const next = (error, socket, session) => sessionSockets.emit('connection', error, socket, session); - io.on 'connection', (socket) -> - req = socket.handshake - cookieParser req, {}, () -> - sessionId = req.signedCookies and req.signedCookies[cookieName] - if not sessionId - return next(missingSessionError, socket) - sessionStore.get sessionId, (error, session) -> - if error - return next(error, socket) - if not session - return next(missingSessionError, socket) - next(null, socket, session) + io.on('connection', function(socket) { + const req = socket.handshake; + return cookieParser(req, {}, function() { + const sessionId = req.signedCookies && req.signedCookies[cookieName]; + if (!sessionId) { + return next(missingSessionError, socket); + } + return sessionStore.get(sessionId, function(error, session) { + if (error) { + return next(error, socket); + } + if (!session) { + return next(missingSessionError, socket); + } + return next(null, socket, session); + }); + }); + }); - return sessionSockets + return sessionSockets; +}; diff --git a/services/real-time/app/coffee/WebApiManager.js b/services/real-time/app/coffee/WebApiManager.js index 3c0551a815..489d4c0a7b 100644 --- a/services/real-time/app/coffee/WebApiManager.js +++ b/services/real-time/app/coffee/WebApiManager.js @@ -1,38 +1,54 @@ -request = require "request" -settings = require "settings-sharelatex" -logger = require "logger-sharelatex" -{ CodedError } = require "./Errors" +/* + * decaffeinate suggestions: + * DS102: Remove unnecessary code created because of implicit returns + * DS207: Consider shorter variations of null checks + * Full docs: https://github.com/decaffeinate/decaffeinate/blob/master/docs/suggestions.md + */ +let WebApiManager; +const request = require("request"); +const settings = require("settings-sharelatex"); +const logger = require("logger-sharelatex"); +const { CodedError } = require("./Errors"); -module.exports = WebApiManager = - joinProject: (project_id, user, callback = (error, project, privilegeLevel, isRestrictedUser) ->) -> - user_id = user._id - logger.log {project_id, user_id}, "sending join project request to web" - url = "#{settings.apis.web.url}/project/#{project_id}/join" - headers = {} - if user.anonymousAccessToken? - headers['x-sl-anonymous-access-token'] = user.anonymousAccessToken - request.post { - url: url - qs: {user_id} - auth: - user: settings.apis.web.user - pass: settings.apis.web.pass +module.exports = (WebApiManager = { + joinProject(project_id, user, callback) { + if (callback == null) { callback = function(error, project, privilegeLevel, isRestrictedUser) {}; } + const user_id = user._id; + logger.log({project_id, user_id}, "sending join project request to web"); + const url = `${settings.apis.web.url}/project/${project_id}/join`; + const headers = {}; + if (user.anonymousAccessToken != null) { + headers['x-sl-anonymous-access-token'] = user.anonymousAccessToken; + } + return request.post({ + url, + qs: {user_id}, + auth: { + user: settings.apis.web.user, + pass: settings.apis.web.pass, sendImmediately: true - json: true - jar: false - headers: headers - }, (error, response, data) -> - return callback(error) if error? - if 200 <= response.statusCode < 300 - if !data? || !data?.project? - err = new Error('no data returned from joinProject request') - logger.error {err, project_id, user_id}, "error accessing web api" - return callback(err) - callback null, data.project, data.privilegeLevel, data.isRestrictedUser - else if response.statusCode == 429 - logger.log(project_id, user_id, "rate-limit hit when joining project") - callback(new CodedError("rate-limit hit when joining project", "TooManyRequests")) - else - err = new Error("non-success status code from web: #{response.statusCode}") - logger.error {err, project_id, user_id}, "error accessing web api" - callback err + }, + json: true, + jar: false, + headers + }, function(error, response, data) { + let err; + if (error != null) { return callback(error); } + if (200 <= response.statusCode && response.statusCode < 300) { + if ((data == null) || ((data != null ? data.project : undefined) == null)) { + err = new Error('no data returned from joinProject request'); + logger.error({err, project_id, user_id}, "error accessing web api"); + return callback(err); + } + return callback(null, data.project, data.privilegeLevel, data.isRestrictedUser); + } else if (response.statusCode === 429) { + logger.log(project_id, user_id, "rate-limit hit when joining project"); + return callback(new CodedError("rate-limit hit when joining project", "TooManyRequests")); + } else { + err = new Error(`non-success status code from web: ${response.statusCode}`); + logger.error({err, project_id, user_id}, "error accessing web api"); + return callback(err); + } + }); + } +}); diff --git a/services/real-time/app/coffee/WebsocketController.js b/services/real-time/app/coffee/WebsocketController.js index f93e67f2a2..dcd6955ac7 100644 --- a/services/real-time/app/coffee/WebsocketController.js +++ b/services/real-time/app/coffee/WebsocketController.js @@ -1,276 +1,354 @@ -logger = require "logger-sharelatex" -metrics = require "metrics-sharelatex" -settings = require "settings-sharelatex" -WebApiManager = require "./WebApiManager" -AuthorizationManager = require "./AuthorizationManager" -DocumentUpdaterManager = require "./DocumentUpdaterManager" -ConnectedUsersManager = require "./ConnectedUsersManager" -WebsocketLoadBalancer = require "./WebsocketLoadBalancer" -RoomManager = require "./RoomManager" +/* + * decaffeinate suggestions: + * DS101: Remove unnecessary use of Array.from + * DS102: Remove unnecessary code created because of implicit returns + * DS103: Rewrite code to no longer use __guard__ + * DS207: Consider shorter variations of null checks + * Full docs: https://github.com/decaffeinate/decaffeinate/blob/master/docs/suggestions.md + */ +let WebsocketController; +const logger = require("logger-sharelatex"); +const metrics = require("metrics-sharelatex"); +const settings = require("settings-sharelatex"); +const WebApiManager = require("./WebApiManager"); +const AuthorizationManager = require("./AuthorizationManager"); +const DocumentUpdaterManager = require("./DocumentUpdaterManager"); +const ConnectedUsersManager = require("./ConnectedUsersManager"); +const WebsocketLoadBalancer = require("./WebsocketLoadBalancer"); +const RoomManager = require("./RoomManager"); -module.exports = WebsocketController = - # If the protocol version changes when the client reconnects, - # it will force a full refresh of the page. Useful for non-backwards - # compatible protocol changes. Use only in extreme need. - PROTOCOL_VERSION: 2 +module.exports = (WebsocketController = { + // If the protocol version changes when the client reconnects, + // it will force a full refresh of the page. Useful for non-backwards + // compatible protocol changes. Use only in extreme need. + PROTOCOL_VERSION: 2, - joinProject: (client, user, project_id, callback = (error, project, privilegeLevel, protocolVersion) ->) -> - if client.disconnected - metrics.inc('editor.join-project.disconnected', 1, {status: 'immediately'}) - return callback() + joinProject(client, user, project_id, callback) { + if (callback == null) { callback = function(error, project, privilegeLevel, protocolVersion) {}; } + if (client.disconnected) { + metrics.inc('editor.join-project.disconnected', 1, {status: 'immediately'}); + return callback(); + } - user_id = user?._id - logger.log {user_id, project_id, client_id: client.id}, "user joining project" - metrics.inc "editor.join-project" - WebApiManager.joinProject project_id, user, (error, project, privilegeLevel, isRestrictedUser) -> - return callback(error) if error? - if client.disconnected - metrics.inc('editor.join-project.disconnected', 1, {status: 'after-web-api-call'}) - return callback() + const user_id = user != null ? user._id : undefined; + logger.log({user_id, project_id, client_id: client.id}, "user joining project"); + metrics.inc("editor.join-project"); + return WebApiManager.joinProject(project_id, user, function(error, project, privilegeLevel, isRestrictedUser) { + if (error != null) { return callback(error); } + if (client.disconnected) { + metrics.inc('editor.join-project.disconnected', 1, {status: 'after-web-api-call'}); + return callback(); + } - if !privilegeLevel or privilegeLevel == "" - err = new Error("not authorized") - logger.warn {err, project_id, user_id, client_id: client.id}, "user is not authorized to join project" - return callback(err) + if (!privilegeLevel || (privilegeLevel === "")) { + const err = new Error("not authorized"); + logger.warn({err, project_id, user_id, client_id: client.id}, "user is not authorized to join project"); + return callback(err); + } - client.ol_context = {} - client.ol_context["privilege_level"] = privilegeLevel - client.ol_context["user_id"] = user_id - client.ol_context["project_id"] = project_id - client.ol_context["owner_id"] = project?.owner?._id - client.ol_context["first_name"] = user?.first_name - client.ol_context["last_name"] = user?.last_name - client.ol_context["email"] = user?.email - client.ol_context["connected_time"] = new Date() - client.ol_context["signup_date"] = user?.signUpDate - client.ol_context["login_count"] = user?.loginCount - client.ol_context["is_restricted_user"] = !!(isRestrictedUser) + client.ol_context = {}; + client.ol_context["privilege_level"] = privilegeLevel; + client.ol_context["user_id"] = user_id; + client.ol_context["project_id"] = project_id; + client.ol_context["owner_id"] = __guard__(project != null ? project.owner : undefined, x => x._id); + client.ol_context["first_name"] = user != null ? user.first_name : undefined; + client.ol_context["last_name"] = user != null ? user.last_name : undefined; + client.ol_context["email"] = user != null ? user.email : undefined; + client.ol_context["connected_time"] = new Date(); + client.ol_context["signup_date"] = user != null ? user.signUpDate : undefined; + client.ol_context["login_count"] = user != null ? user.loginCount : undefined; + client.ol_context["is_restricted_user"] = !!(isRestrictedUser); - RoomManager.joinProject client, project_id, (err) -> - return callback(err) if err - logger.log {user_id, project_id, client_id: client.id}, "user joined project" - callback null, project, privilegeLevel, WebsocketController.PROTOCOL_VERSION + RoomManager.joinProject(client, project_id, function(err) { + if (err) { return callback(err); } + logger.log({user_id, project_id, client_id: client.id}, "user joined project"); + return callback(null, project, privilegeLevel, WebsocketController.PROTOCOL_VERSION); + }); - # No need to block for setting the user as connected in the cursor tracking - ConnectedUsersManager.updateUserPosition project_id, client.publicId, user, null, () -> + // No need to block for setting the user as connected in the cursor tracking + return ConnectedUsersManager.updateUserPosition(project_id, client.publicId, user, null, function() {}); + }); + }, - # We want to flush a project if there are no more (local) connected clients - # but we need to wait for the triggering client to disconnect. How long we wait - # is determined by FLUSH_IF_EMPTY_DELAY. - FLUSH_IF_EMPTY_DELAY: 500 #ms - leaveProject: (io, client, callback = (error) ->) -> - {project_id, user_id} = client.ol_context - return callback() unless project_id # client did not join project + // We want to flush a project if there are no more (local) connected clients + // but we need to wait for the triggering client to disconnect. How long we wait + // is determined by FLUSH_IF_EMPTY_DELAY. + FLUSH_IF_EMPTY_DELAY: 500, //ms + leaveProject(io, client, callback) { + if (callback == null) { callback = function(error) {}; } + const {project_id, user_id} = client.ol_context; + if (!project_id) { return callback(); } // client did not join project - metrics.inc "editor.leave-project" - logger.log {project_id, user_id, client_id: client.id}, "client leaving project" - WebsocketLoadBalancer.emitToRoom project_id, "clientTracking.clientDisconnected", client.publicId + metrics.inc("editor.leave-project"); + logger.log({project_id, user_id, client_id: client.id}, "client leaving project"); + WebsocketLoadBalancer.emitToRoom(project_id, "clientTracking.clientDisconnected", client.publicId); - # We can do this in the background - ConnectedUsersManager.markUserAsDisconnected project_id, client.publicId, (err) -> - if err? - logger.error {err, project_id, user_id, client_id: client.id}, "error marking client as disconnected" + // We can do this in the background + ConnectedUsersManager.markUserAsDisconnected(project_id, client.publicId, function(err) { + if (err != null) { + return logger.error({err, project_id, user_id, client_id: client.id}, "error marking client as disconnected"); + } + }); - RoomManager.leaveProjectAndDocs(client) - setTimeout () -> - remainingClients = io.sockets.clients(project_id) - if remainingClients.length == 0 - # Flush project in the background - DocumentUpdaterManager.flushProjectToMongoAndDelete project_id, (err) -> - if err? - logger.error {err, project_id, user_id, client_id: client.id}, "error flushing to doc updater after leaving project" - callback() - , WebsocketController.FLUSH_IF_EMPTY_DELAY + RoomManager.leaveProjectAndDocs(client); + return setTimeout(function() { + const remainingClients = io.sockets.clients(project_id); + if (remainingClients.length === 0) { + // Flush project in the background + DocumentUpdaterManager.flushProjectToMongoAndDelete(project_id, function(err) { + if (err != null) { + return logger.error({err, project_id, user_id, client_id: client.id}, "error flushing to doc updater after leaving project"); + } + }); + } + return callback(); + } + , WebsocketController.FLUSH_IF_EMPTY_DELAY); + }, - joinDoc: (client, doc_id, fromVersion = -1, options, callback = (error, doclines, version, ops, ranges) ->) -> - if client.disconnected - metrics.inc('editor.join-doc.disconnected', 1, {status: 'immediately'}) - return callback() + joinDoc(client, doc_id, fromVersion, options, callback) { + if (fromVersion == null) { fromVersion = -1; } + if (callback == null) { callback = function(error, doclines, version, ops, ranges) {}; } + if (client.disconnected) { + metrics.inc('editor.join-doc.disconnected', 1, {status: 'immediately'}); + return callback(); + } - metrics.inc "editor.join-doc" - {project_id, user_id, is_restricted_user} = client.ol_context - return callback(new Error("no project_id found on client")) if !project_id? - logger.log {user_id, project_id, doc_id, fromVersion, client_id: client.id}, "client joining doc" + metrics.inc("editor.join-doc"); + const {project_id, user_id, is_restricted_user} = client.ol_context; + if ((project_id == null)) { return callback(new Error("no project_id found on client")); } + logger.log({user_id, project_id, doc_id, fromVersion, client_id: client.id}, "client joining doc"); - AuthorizationManager.assertClientCanViewProject client, (error) -> - return callback(error) if error? - # ensure the per-doc applied-ops channel is subscribed before sending the - # doc to the client, so that no events are missed. - RoomManager.joinDoc client, doc_id, (error) -> - return callback(error) if error? - if client.disconnected - metrics.inc('editor.join-doc.disconnected', 1, {status: 'after-joining-room'}) - # the client will not read the response anyways - return callback() + return AuthorizationManager.assertClientCanViewProject(client, function(error) { + if (error != null) { return callback(error); } + // ensure the per-doc applied-ops channel is subscribed before sending the + // doc to the client, so that no events are missed. + return RoomManager.joinDoc(client, doc_id, function(error) { + if (error != null) { return callback(error); } + if (client.disconnected) { + metrics.inc('editor.join-doc.disconnected', 1, {status: 'after-joining-room'}); + // the client will not read the response anyways + return callback(); + } - DocumentUpdaterManager.getDocument project_id, doc_id, fromVersion, (error, lines, version, ranges, ops) -> - return callback(error) if error? - if client.disconnected - metrics.inc('editor.join-doc.disconnected', 1, {status: 'after-doc-updater-call'}) - # the client will not read the response anyways - return callback() + return DocumentUpdaterManager.getDocument(project_id, doc_id, fromVersion, function(error, lines, version, ranges, ops) { + let err; + if (error != null) { return callback(error); } + if (client.disconnected) { + metrics.inc('editor.join-doc.disconnected', 1, {status: 'after-doc-updater-call'}); + // the client will not read the response anyways + return callback(); + } - if is_restricted_user and ranges?.comments? - ranges.comments = [] + if (is_restricted_user && ((ranges != null ? ranges.comments : undefined) != null)) { + ranges.comments = []; + } - # Encode any binary bits of data so it can go via WebSockets - # See http://ecmanaut.blogspot.co.uk/2006/07/encoding-decoding-utf8-in-javascript.html - encodeForWebsockets = (text) -> unescape(encodeURIComponent(text)) - escapedLines = [] - for line in lines - try - line = encodeForWebsockets(line) - catch err - logger.err {err, project_id, doc_id, fromVersion, line, client_id: client.id}, "error encoding line uri component" - return callback(err) - escapedLines.push line - if options.encodeRanges - try - for comment in ranges?.comments or [] - comment.op.c = encodeForWebsockets(comment.op.c) if comment.op.c? - for change in ranges?.changes or [] - change.op.i = encodeForWebsockets(change.op.i) if change.op.i? - change.op.d = encodeForWebsockets(change.op.d) if change.op.d? - catch err - logger.err {err, project_id, doc_id, fromVersion, ranges, client_id: client.id}, "error encoding range uri component" - return callback(err) + // Encode any binary bits of data so it can go via WebSockets + // See http://ecmanaut.blogspot.co.uk/2006/07/encoding-decoding-utf8-in-javascript.html + const encodeForWebsockets = text => unescape(encodeURIComponent(text)); + const escapedLines = []; + for (let line of Array.from(lines)) { + try { + line = encodeForWebsockets(line); + } catch (error1) { + err = error1; + logger.err({err, project_id, doc_id, fromVersion, line, client_id: client.id}, "error encoding line uri component"); + return callback(err); + } + escapedLines.push(line); + } + if (options.encodeRanges) { + try { + for (let comment of Array.from((ranges != null ? ranges.comments : undefined) || [])) { + if (comment.op.c != null) { comment.op.c = encodeForWebsockets(comment.op.c); } + } + for (let change of Array.from((ranges != null ? ranges.changes : undefined) || [])) { + if (change.op.i != null) { change.op.i = encodeForWebsockets(change.op.i); } + if (change.op.d != null) { change.op.d = encodeForWebsockets(change.op.d); } + } + } catch (error2) { + err = error2; + logger.err({err, project_id, doc_id, fromVersion, ranges, client_id: client.id}, "error encoding range uri component"); + return callback(err); + } + } - AuthorizationManager.addAccessToDoc client, doc_id - logger.log {user_id, project_id, doc_id, fromVersion, client_id: client.id}, "client joined doc" - callback null, escapedLines, version, ops, ranges + AuthorizationManager.addAccessToDoc(client, doc_id); + logger.log({user_id, project_id, doc_id, fromVersion, client_id: client.id}, "client joined doc"); + return callback(null, escapedLines, version, ops, ranges); + }); + }); + }); + }, - leaveDoc: (client, doc_id, callback = (error) ->) -> - # client may have disconnected, but we have to cleanup internal state. - metrics.inc "editor.leave-doc" - {project_id, user_id} = client.ol_context - logger.log {user_id, project_id, doc_id, client_id: client.id}, "client leaving doc" - RoomManager.leaveDoc(client, doc_id) - # we could remove permission when user leaves a doc, but because - # the connection is per-project, we continue to allow access - # after the initial joinDoc since we know they are already authorised. - ## AuthorizationManager.removeAccessToDoc client, doc_id - callback() - updateClientPosition: (client, cursorData, callback = (error) ->) -> - if client.disconnected - # do not create a ghost entry in redis - return callback() + leaveDoc(client, doc_id, callback) { + // client may have disconnected, but we have to cleanup internal state. + if (callback == null) { callback = function(error) {}; } + metrics.inc("editor.leave-doc"); + const {project_id, user_id} = client.ol_context; + logger.log({user_id, project_id, doc_id, client_id: client.id}, "client leaving doc"); + RoomManager.leaveDoc(client, doc_id); + // we could remove permission when user leaves a doc, but because + // the connection is per-project, we continue to allow access + // after the initial joinDoc since we know they are already authorised. + //# AuthorizationManager.removeAccessToDoc client, doc_id + return callback(); + }, + updateClientPosition(client, cursorData, callback) { + if (callback == null) { callback = function(error) {}; } + if (client.disconnected) { + // do not create a ghost entry in redis + return callback(); + } - metrics.inc "editor.update-client-position", 0.1 - {project_id, first_name, last_name, email, user_id} = client.ol_context - logger.log {user_id, project_id, client_id: client.id, cursorData: cursorData}, "updating client position" + metrics.inc("editor.update-client-position", 0.1); + const {project_id, first_name, last_name, email, user_id} = client.ol_context; + logger.log({user_id, project_id, client_id: client.id, cursorData}, "updating client position"); - AuthorizationManager.assertClientCanViewProjectAndDoc client, cursorData.doc_id, (error) -> - if error? - logger.warn {err: error, client_id: client.id, project_id, user_id}, "silently ignoring unauthorized updateClientPosition. Client likely hasn't called joinProject yet." - return callback() - cursorData.id = client.publicId - cursorData.user_id = user_id if user_id? - cursorData.email = email if email? - # Don't store anonymous users in redis to avoid influx - if !user_id or user_id == 'anonymous-user' - cursorData.name = "" - callback() - else - cursorData.name = if first_name && last_name - "#{first_name} #{last_name}" - else if first_name + return AuthorizationManager.assertClientCanViewProjectAndDoc(client, cursorData.doc_id, function(error) { + if (error != null) { + logger.warn({err: error, client_id: client.id, project_id, user_id}, "silently ignoring unauthorized updateClientPosition. Client likely hasn't called joinProject yet."); + return callback(); + } + cursorData.id = client.publicId; + if (user_id != null) { cursorData.user_id = user_id; } + if (email != null) { cursorData.email = email; } + // Don't store anonymous users in redis to avoid influx + if (!user_id || (user_id === 'anonymous-user')) { + cursorData.name = ""; + callback(); + } else { + cursorData.name = first_name && last_name ? + `${first_name} ${last_name}` + : first_name ? first_name - else if last_name + : last_name ? last_name - else - "" + : + ""; ConnectedUsersManager.updateUserPosition(project_id, client.publicId, { - first_name: first_name, - last_name: last_name, - email: email, + first_name, + last_name, + email, _id: user_id }, { row: cursorData.row, column: cursorData.column, doc_id: cursorData.doc_id - }, callback) - WebsocketLoadBalancer.emitToRoom(project_id, "clientTracking.clientUpdated", cursorData) + }, callback); + } + return WebsocketLoadBalancer.emitToRoom(project_id, "clientTracking.clientUpdated", cursorData); + }); + }, - CLIENT_REFRESH_DELAY: 1000 - getConnectedUsers: (client, callback = (error, users) ->) -> - if client.disconnected - # they are not interested anymore, skip the redis lookups - return callback() + CLIENT_REFRESH_DELAY: 1000, + getConnectedUsers(client, callback) { + if (callback == null) { callback = function(error, users) {}; } + if (client.disconnected) { + // they are not interested anymore, skip the redis lookups + return callback(); + } - metrics.inc "editor.get-connected-users" - {project_id, user_id, is_restricted_user} = client.ol_context - if is_restricted_user - return callback(null, []) - return callback(new Error("no project_id found on client")) if !project_id? - logger.log {user_id, project_id, client_id: client.id}, "getting connected users" - AuthorizationManager.assertClientCanViewProject client, (error) -> - return callback(error) if error? - WebsocketLoadBalancer.emitToRoom project_id, 'clientTracking.refresh' - setTimeout () -> - ConnectedUsersManager.getConnectedUsers project_id, (error, users) -> - return callback(error) if error? - callback null, users - logger.log {user_id, project_id, client_id: client.id}, "got connected users" - , WebsocketController.CLIENT_REFRESH_DELAY + metrics.inc("editor.get-connected-users"); + const {project_id, user_id, is_restricted_user} = client.ol_context; + if (is_restricted_user) { + return callback(null, []); + } + if ((project_id == null)) { return callback(new Error("no project_id found on client")); } + logger.log({user_id, project_id, client_id: client.id}, "getting connected users"); + return AuthorizationManager.assertClientCanViewProject(client, function(error) { + if (error != null) { return callback(error); } + WebsocketLoadBalancer.emitToRoom(project_id, 'clientTracking.refresh'); + return setTimeout(() => ConnectedUsersManager.getConnectedUsers(project_id, function(error, users) { + if (error != null) { return callback(error); } + callback(null, users); + return logger.log({user_id, project_id, client_id: client.id}, "got connected users"); + }) + , WebsocketController.CLIENT_REFRESH_DELAY); + }); + }, - applyOtUpdate: (client, doc_id, update, callback = (error) ->) -> - # client may have disconnected, but we can submit their update to doc-updater anyways. - {user_id, project_id} = client.ol_context - return callback(new Error("no project_id found on client")) if !project_id? + applyOtUpdate(client, doc_id, update, callback) { + // client may have disconnected, but we can submit their update to doc-updater anyways. + if (callback == null) { callback = function(error) {}; } + const {user_id, project_id} = client.ol_context; + if ((project_id == null)) { return callback(new Error("no project_id found on client")); } - WebsocketController._assertClientCanApplyUpdate client, doc_id, update, (error) -> - if error? - logger.warn {err: error, doc_id, client_id: client.id, version: update.v}, "client is not authorized to make update" - setTimeout () -> - # Disconnect, but give the client the chance to receive the error - client.disconnect() - , 100 - return callback(error) - update.meta ||= {} - update.meta.source = client.publicId - update.meta.user_id = user_id - metrics.inc "editor.doc-update", 0.3 + return WebsocketController._assertClientCanApplyUpdate(client, doc_id, update, function(error) { + if (error != null) { + logger.warn({err: error, doc_id, client_id: client.id, version: update.v}, "client is not authorized to make update"); + setTimeout(() => // Disconnect, but give the client the chance to receive the error + client.disconnect() + , 100); + return callback(error); + } + if (!update.meta) { update.meta = {}; } + update.meta.source = client.publicId; + update.meta.user_id = user_id; + metrics.inc("editor.doc-update", 0.3); - logger.log {user_id, doc_id, project_id, client_id: client.id, version: update.v}, "sending update to doc updater" + logger.log({user_id, doc_id, project_id, client_id: client.id, version: update.v}, "sending update to doc updater"); - DocumentUpdaterManager.queueChange project_id, doc_id, update, (error) -> - if error?.message == "update is too large" - metrics.inc "update_too_large" - updateSize = error.updateSize - logger.warn({user_id, project_id, doc_id, updateSize}, "update is too large") + return DocumentUpdaterManager.queueChange(project_id, doc_id, update, function(error) { + if ((error != null ? error.message : undefined) === "update is too large") { + metrics.inc("update_too_large"); + const { + updateSize + } = error; + logger.warn({user_id, project_id, doc_id, updateSize}, "update is too large"); - # mark the update as received -- the client should not send it again! - callback() + // mark the update as received -- the client should not send it again! + callback(); - # trigger an out-of-sync error - message = {project_id, doc_id, error: "update is too large"} - setTimeout () -> - if client.disconnected - # skip the message broadcast, the client has moved on - return metrics.inc('editor.doc-update.disconnected', 1, {status:'at-otUpdateError'}) - client.emit "otUpdateError", message.error, message - client.disconnect() - , 100 - return + // trigger an out-of-sync error + const message = {project_id, doc_id, error: "update is too large"}; + setTimeout(function() { + if (client.disconnected) { + // skip the message broadcast, the client has moved on + return metrics.inc('editor.doc-update.disconnected', 1, {status:'at-otUpdateError'}); + } + client.emit("otUpdateError", message.error, message); + return client.disconnect(); + } + , 100); + return; + } - if error? - logger.error {err: error, project_id, doc_id, client_id: client.id, version: update.v}, "document was not available for update" - client.disconnect() - callback(error) + if (error != null) { + logger.error({err: error, project_id, doc_id, client_id: client.id, version: update.v}, "document was not available for update"); + client.disconnect(); + } + return callback(error); + }); + }); + }, - _assertClientCanApplyUpdate: (client, doc_id, update, callback) -> - AuthorizationManager.assertClientCanEditProjectAndDoc client, doc_id, (error) -> - if error? - if error.message == "not authorized" and WebsocketController._isCommentUpdate(update) - # This might be a comment op, which we only need read-only priveleges for - AuthorizationManager.assertClientCanViewProjectAndDoc client, doc_id, callback - else - return callback(error) - else - return callback(null) + _assertClientCanApplyUpdate(client, doc_id, update, callback) { + return AuthorizationManager.assertClientCanEditProjectAndDoc(client, doc_id, function(error) { + if (error != null) { + if ((error.message === "not authorized") && WebsocketController._isCommentUpdate(update)) { + // This might be a comment op, which we only need read-only priveleges for + return AuthorizationManager.assertClientCanViewProjectAndDoc(client, doc_id, callback); + } else { + return callback(error); + } + } else { + return callback(null); + } + }); + }, - _isCommentUpdate: (update) -> - for op in update.op - if !op.c? - return false - return true + _isCommentUpdate(update) { + for (let op of Array.from(update.op)) { + if ((op.c == null)) { + return false; + } + } + return true; + } +}); + +function __guard__(value, transform) { + return (typeof value !== 'undefined' && value !== null) ? transform(value) : undefined; +} \ No newline at end of file diff --git a/services/real-time/app/coffee/WebsocketLoadBalancer.js b/services/real-time/app/coffee/WebsocketLoadBalancer.js index 209ec0bb08..0734929453 100644 --- a/services/real-time/app/coffee/WebsocketLoadBalancer.js +++ b/services/real-time/app/coffee/WebsocketLoadBalancer.js @@ -1,14 +1,23 @@ -Settings = require 'settings-sharelatex' -logger = require 'logger-sharelatex' -RedisClientManager = require "./RedisClientManager" -SafeJsonParse = require "./SafeJsonParse" -EventLogger = require "./EventLogger" -HealthCheckManager = require "./HealthCheckManager" -RoomManager = require "./RoomManager" -ChannelManager = require "./ChannelManager" -ConnectedUsersManager = require "./ConnectedUsersManager" +/* + * decaffeinate suggestions: + * DS101: Remove unnecessary use of Array.from + * DS102: Remove unnecessary code created because of implicit returns + * DS205: Consider reworking code to avoid use of IIFEs + * DS207: Consider shorter variations of null checks + * Full docs: https://github.com/decaffeinate/decaffeinate/blob/master/docs/suggestions.md + */ +let WebsocketLoadBalancer; +const Settings = require('settings-sharelatex'); +const logger = require('logger-sharelatex'); +const RedisClientManager = require("./RedisClientManager"); +const SafeJsonParse = require("./SafeJsonParse"); +const EventLogger = require("./EventLogger"); +const HealthCheckManager = require("./HealthCheckManager"); +const RoomManager = require("./RoomManager"); +const ChannelManager = require("./ChannelManager"); +const ConnectedUsersManager = require("./ConnectedUsersManager"); -RESTRICTED_USER_MESSAGE_TYPE_PASS_LIST = [ +const RESTRICTED_USER_MESSAGE_TYPE_PASS_LIST = [ 'connectionAccepted', 'otUpdateApplied', 'otUpdateError', @@ -17,88 +26,126 @@ RESTRICTED_USER_MESSAGE_TYPE_PASS_LIST = [ 'reciveNewFile', 'reciveNewFolder', 'removeEntity' -] +]; -module.exports = WebsocketLoadBalancer = - rclientPubList: RedisClientManager.createClientList(Settings.redis.pubsub) - rclientSubList: RedisClientManager.createClientList(Settings.redis.pubsub) +module.exports = (WebsocketLoadBalancer = { + rclientPubList: RedisClientManager.createClientList(Settings.redis.pubsub), + rclientSubList: RedisClientManager.createClientList(Settings.redis.pubsub), - emitToRoom: (room_id, message, payload...) -> - if !room_id? - logger.warn {message, payload}, "no room_id provided, ignoring emitToRoom" - return - data = JSON.stringify - room_id: room_id - message: message - payload: payload - logger.log {room_id, message, payload, length: data.length}, "emitting to room" + emitToRoom(room_id, message, ...payload) { + if ((room_id == null)) { + logger.warn({message, payload}, "no room_id provided, ignoring emitToRoom"); + return; + } + const data = JSON.stringify({ + room_id, + message, + payload + }); + logger.log({room_id, message, payload, length: data.length}, "emitting to room"); - for rclientPub in @rclientPubList - ChannelManager.publish rclientPub, "editor-events", room_id, data + return Array.from(this.rclientPubList).map((rclientPub) => + ChannelManager.publish(rclientPub, "editor-events", room_id, data)); + }, - emitToAll: (message, payload...) -> - @emitToRoom "all", message, payload... + emitToAll(message, ...payload) { + return this.emitToRoom("all", message, ...Array.from(payload)); + }, - listenForEditorEvents: (io) -> - logger.log {rclients: @rclientPubList.length}, "publishing editor events" - logger.log {rclients: @rclientSubList.length}, "listening for editor events" - for rclientSub in @rclientSubList - rclientSub.subscribe "editor-events" - rclientSub.on "message", (channel, message) -> - EventLogger.debugEvent(channel, message) if Settings.debugEvents > 0 - WebsocketLoadBalancer._processEditorEvent io, channel, message - @handleRoomUpdates(@rclientSubList) + listenForEditorEvents(io) { + logger.log({rclients: this.rclientPubList.length}, "publishing editor events"); + logger.log({rclients: this.rclientSubList.length}, "listening for editor events"); + for (let rclientSub of Array.from(this.rclientSubList)) { + rclientSub.subscribe("editor-events"); + rclientSub.on("message", function(channel, message) { + if (Settings.debugEvents > 0) { EventLogger.debugEvent(channel, message); } + return WebsocketLoadBalancer._processEditorEvent(io, channel, message); + }); + } + return this.handleRoomUpdates(this.rclientSubList); + }, - handleRoomUpdates: (rclientSubList) -> - roomEvents = RoomManager.eventSource() - roomEvents.on 'project-active', (project_id) -> - subscribePromises = for rclient in rclientSubList - ChannelManager.subscribe rclient, "editor-events", project_id - RoomManager.emitOnCompletion(subscribePromises, "project-subscribed-#{project_id}") - roomEvents.on 'project-empty', (project_id) -> - for rclient in rclientSubList - ChannelManager.unsubscribe rclient, "editor-events", project_id + handleRoomUpdates(rclientSubList) { + const roomEvents = RoomManager.eventSource(); + roomEvents.on('project-active', function(project_id) { + const subscribePromises = Array.from(rclientSubList).map((rclient) => + ChannelManager.subscribe(rclient, "editor-events", project_id)); + return RoomManager.emitOnCompletion(subscribePromises, `project-subscribed-${project_id}`); + }); + return roomEvents.on('project-empty', project_id => Array.from(rclientSubList).map((rclient) => + ChannelManager.unsubscribe(rclient, "editor-events", project_id))); + }, - _processEditorEvent: (io, channel, message) -> - SafeJsonParse.parse message, (error, message) -> - if error? - logger.error {err: error, channel}, "error parsing JSON" - return - if message.room_id == "all" - io.sockets.emit(message.message, message.payload...) - else if message.message is 'clientTracking.refresh' && message.room_id? + _processEditorEvent(io, channel, message) { + return SafeJsonParse.parse(message, function(error, message) { + let clientList; + let client; + if (error != null) { + logger.error({err: error, channel}, "error parsing JSON"); + return; + } + if (message.room_id === "all") { + return io.sockets.emit(message.message, ...Array.from(message.payload)); + } else if ((message.message === 'clientTracking.refresh') && (message.room_id != null)) { + clientList = io.sockets.clients(message.room_id); + logger.log({channel, message: message.message, room_id: message.room_id, message_id: message._id, socketIoClients: ((() => { + const result = []; + for (client of Array.from(clientList)) { result.push(client.id); + } + return result; + })())}, "refreshing client list"); + return (() => { + const result1 = []; + for (client of Array.from(clientList)) { + result1.push(ConnectedUsersManager.refreshClient(message.room_id, client.publicId)); + } + return result1; + })(); + } else if (message.room_id != null) { + if ((message._id != null) && Settings.checkEventOrder) { + const status = EventLogger.checkEventOrder("editor-events", message._id, message); + if (status === "duplicate") { + return; // skip duplicate events + } + } + + const is_restricted_message = !Array.from(RESTRICTED_USER_MESSAGE_TYPE_PASS_LIST).includes(message.message); + + // send messages only to unique clients (due to duplicate entries in io.sockets.clients) clientList = io.sockets.clients(message.room_id) - logger.log {channel:channel, message: message.message, room_id: message.room_id, message_id: message._id, socketIoClients: (client.id for client in clientList)}, "refreshing client list" - for client in clientList - ConnectedUsersManager.refreshClient(message.room_id, client.publicId) - else if message.room_id? - if message._id? && Settings.checkEventOrder - status = EventLogger.checkEventOrder("editor-events", message._id, message) - if status is "duplicate" - return # skip duplicate events + .filter(client => !(is_restricted_message && client.ol_context['is_restricted_user'])); - is_restricted_message = message.message not in RESTRICTED_USER_MESSAGE_TYPE_PASS_LIST - - # send messages only to unique clients (due to duplicate entries in io.sockets.clients) - clientList = io.sockets.clients(message.room_id) - .filter((client) -> - !(is_restricted_message && client.ol_context['is_restricted_user']) - ) - - # avoid unnecessary work if no clients are connected - return if clientList.length is 0 - logger.log { - channel: channel, + // avoid unnecessary work if no clients are connected + if (clientList.length === 0) { return; } + logger.log({ + channel, message: message.message, room_id: message.room_id, message_id: message._id, - socketIoClients: (client.id for client in clientList) - }, "distributing event to clients" - seen = {} - for client in clientList - if !seen[client.id] - seen[client.id] = true - client.emit(message.message, message.payload...) - else if message.health_check? - logger.debug {message}, "got health check message in editor events channel" - HealthCheckManager.check channel, message.key + socketIoClients: ((() => { + const result2 = []; + for (client of Array.from(clientList)) { result2.push(client.id); + } + return result2; + })()) + }, "distributing event to clients"); + const seen = {}; + return (() => { + const result3 = []; + for (client of Array.from(clientList)) { + if (!seen[client.id]) { + seen[client.id] = true; + result3.push(client.emit(message.message, ...Array.from(message.payload))); + } else { + result3.push(undefined); + } + } + return result3; + })(); + } else if (message.health_check != null) { + logger.debug({message}, "got health check message in editor events channel"); + return HealthCheckManager.check(channel, message.key); + } + }); + } +}); From a397154e182c7f2f583511dfc5acb0e8909c1e95 Mon Sep 17 00:00:00 2001 From: decaffeinate Date: Tue, 23 Jun 2020 18:29:38 +0100 Subject: [PATCH 370/491] decaffeinate: Run post-processing cleanups on AuthorizationManager.coffee and 18 other files --- .../app/coffee/AuthorizationManager.js | 8 +++- .../real-time/app/coffee/ChannelManager.js | 5 ++ .../app/coffee/ConnectedUsersManager.js | 6 +++ .../app/coffee/DocumentUpdaterController.js | 8 +++- .../app/coffee/DocumentUpdaterManager.js | 11 ++++- services/real-time/app/coffee/DrainManager.js | 7 ++- services/real-time/app/coffee/Errors.js | 6 +++ services/real-time/app/coffee/EventLogger.js | 7 ++- .../app/coffee/HealthCheckManager.js | 12 ++++- .../real-time/app/coffee/HttpApiController.js | 8 +++- .../real-time/app/coffee/HttpController.js | 8 +++- .../app/coffee/RedisClientManager.js | 7 ++- services/real-time/app/coffee/RoomManager.js | 10 +++- services/real-time/app/coffee/Router.js | 9 +++- .../real-time/app/coffee/SafeJsonParse.js | 5 ++ .../real-time/app/coffee/SessionSockets.js | 2 + .../real-time/app/coffee/WebApiManager.js | 9 +++- .../app/coffee/WebsocketController.js | 46 ++++++++++--------- .../app/coffee/WebsocketLoadBalancer.js | 9 +++- 19 files changed, 145 insertions(+), 38 deletions(-) diff --git a/services/real-time/app/coffee/AuthorizationManager.js b/services/real-time/app/coffee/AuthorizationManager.js index 0ce4c313e2..41caee9ef2 100644 --- a/services/real-time/app/coffee/AuthorizationManager.js +++ b/services/real-time/app/coffee/AuthorizationManager.js @@ -1,3 +1,9 @@ +/* eslint-disable + camelcase, + handle-callback-err, +*/ +// TODO: This file was created by bulk-decaffeinate. +// Fix any style issues and re-enable lint. /* * decaffeinate suggestions: * DS101: Remove unnecessary use of Array.from @@ -19,7 +25,7 @@ module.exports = (AuthorizationManager = { _assertClientHasPrivilegeLevel(client, allowedLevels, callback) { if (callback == null) { callback = function(error) {}; } - if (Array.from(allowedLevels).includes(client.ol_context["privilege_level"])) { + if (Array.from(allowedLevels).includes(client.ol_context.privilege_level)) { return callback(null); } else { return callback(new Error("not authorized")); diff --git a/services/real-time/app/coffee/ChannelManager.js b/services/real-time/app/coffee/ChannelManager.js index eb73802a07..60e7c5c635 100644 --- a/services/real-time/app/coffee/ChannelManager.js +++ b/services/real-time/app/coffee/ChannelManager.js @@ -1,3 +1,8 @@ +/* eslint-disable + no-unused-vars, +*/ +// TODO: This file was created by bulk-decaffeinate. +// Fix any style issues and re-enable lint. /* * decaffeinate suggestions: * DS102: Remove unnecessary code created because of implicit returns diff --git a/services/real-time/app/coffee/ConnectedUsersManager.js b/services/real-time/app/coffee/ConnectedUsersManager.js index bfdcf608a0..b2a0fc2eee 100644 --- a/services/real-time/app/coffee/ConnectedUsersManager.js +++ b/services/real-time/app/coffee/ConnectedUsersManager.js @@ -1,3 +1,9 @@ +/* eslint-disable + camelcase, + handle-callback-err, +*/ +// TODO: This file was created by bulk-decaffeinate. +// Fix any style issues and re-enable lint. /* * decaffeinate suggestions: * DS102: Remove unnecessary code created because of implicit returns diff --git a/services/real-time/app/coffee/DocumentUpdaterController.js b/services/real-time/app/coffee/DocumentUpdaterController.js index 85078219b6..cbf5c600fd 100644 --- a/services/real-time/app/coffee/DocumentUpdaterController.js +++ b/services/real-time/app/coffee/DocumentUpdaterController.js @@ -1,3 +1,9 @@ +/* eslint-disable + camelcase, + no-unused-vars, +*/ +// TODO: This file was created by bulk-decaffeinate. +// Fix any style issues and re-enable lint. /* * decaffeinate suggestions: * DS101: Remove unnecessary use of Array.from @@ -123,7 +129,7 @@ module.exports = (DocumentUpdaterController = { _processErrorFromDocumentUpdater(io, doc_id, error, message) { return (() => { const result = []; - for (let client of Array.from(io.sockets.clients(doc_id))) { + for (const client of Array.from(io.sockets.clients(doc_id))) { logger.warn({err: error, doc_id, client_id: client.id}, "error from document updater, disconnecting client"); client.emit("otUpdateError", error, message); result.push(client.disconnect()); diff --git a/services/real-time/app/coffee/DocumentUpdaterManager.js b/services/real-time/app/coffee/DocumentUpdaterManager.js index 4b07b8f381..dc5865db62 100644 --- a/services/real-time/app/coffee/DocumentUpdaterManager.js +++ b/services/real-time/app/coffee/DocumentUpdaterManager.js @@ -1,3 +1,10 @@ +/* eslint-disable + camelcase, + handle-callback-err, + no-unused-vars, +*/ +// TODO: This file was created by bulk-decaffeinate. +// Fix any style issues and re-enable lint. /* * decaffeinate suggestions: * DS102: Remove unnecessary code created because of implicit returns @@ -26,7 +33,7 @@ module.exports = (DocumentUpdaterManager = { logger.error({err, url, project_id, doc_id}, "error getting doc from doc updater"); return callback(err); } - if (200 <= res.statusCode && res.statusCode < 300) { + if (res.statusCode >= 200 && res.statusCode < 300) { logger.log({project_id, doc_id}, "got doc from document document updater"); try { body = JSON.parse(body); @@ -61,7 +68,7 @@ module.exports = (DocumentUpdaterManager = { if (err != null) { logger.error({err, project_id}, "error deleting project from document updater"); return callback(err); - } else if (200 <= res.statusCode && res.statusCode < 300) { + } else if (res.statusCode >= 200 && res.statusCode < 300) { logger.log({project_id}, "deleted project from document updater"); return callback(null); } else { diff --git a/services/real-time/app/coffee/DrainManager.js b/services/real-time/app/coffee/DrainManager.js index 2f4067cc3c..466c80fd0c 100644 --- a/services/real-time/app/coffee/DrainManager.js +++ b/services/real-time/app/coffee/DrainManager.js @@ -1,3 +1,8 @@ +/* eslint-disable + no-return-assign, +*/ +// TODO: This file was created by bulk-decaffeinate. +// Fix any style issues and re-enable lint. /* * decaffeinate suggestions: * DS101: Remove unnecessary use of Array.from @@ -38,7 +43,7 @@ module.exports = (DrainManager = { RECONNECTED_CLIENTS: {}, reconnectNClients(io, N) { let drainedCount = 0; - for (let client of Array.from(io.sockets.clients())) { + for (const client of Array.from(io.sockets.clients())) { if (!this.RECONNECTED_CLIENTS[client.id]) { this.RECONNECTED_CLIENTS[client.id] = true; logger.log({client_id: client.id}, "Asking client to reconnect gracefully"); diff --git a/services/real-time/app/coffee/Errors.js b/services/real-time/app/coffee/Errors.js index 2ae4fbd6ab..04437742fb 100644 --- a/services/real-time/app/coffee/Errors.js +++ b/services/real-time/app/coffee/Errors.js @@ -1,3 +1,9 @@ +/* eslint-disable + no-proto, + no-unused-vars, +*/ +// TODO: This file was created by bulk-decaffeinate. +// Fix any style issues and re-enable lint. let Errors; var CodedError = function(message, code) { const error = new Error(message); diff --git a/services/real-time/app/coffee/EventLogger.js b/services/real-time/app/coffee/EventLogger.js index bc01011687..8a700326b5 100644 --- a/services/real-time/app/coffee/EventLogger.js +++ b/services/real-time/app/coffee/EventLogger.js @@ -1,3 +1,8 @@ +/* eslint-disable + camelcase, +*/ +// TODO: This file was created by bulk-decaffeinate. +// Fix any style issues and re-enable lint. /* * decaffeinate suggestions: * DS102: Remove unnecessary code created because of implicit returns @@ -73,7 +78,7 @@ module.exports = (EventLogger = { _cleanEventStream(now) { return (() => { const result = []; - for (let key in EVENT_LOG_TIMESTAMP) { + for (const key in EVENT_LOG_TIMESTAMP) { const timestamp = EVENT_LOG_TIMESTAMP[key]; if ((now - timestamp) > EventLogger.MAX_STALE_TIME_IN_MS) { delete EVENT_LOG_COUNTER[key]; diff --git a/services/real-time/app/coffee/HealthCheckManager.js b/services/real-time/app/coffee/HealthCheckManager.js index 47da253993..f8a9aa672e 100644 --- a/services/real-time/app/coffee/HealthCheckManager.js +++ b/services/real-time/app/coffee/HealthCheckManager.js @@ -1,3 +1,9 @@ +/* eslint-disable + no-return-assign, + no-unused-vars, +*/ +// TODO: This file was created by bulk-decaffeinate. +// Fix any style issues and re-enable lint. /* * decaffeinate suggestions: * DS102: Remove unnecessary code created because of implicit returns @@ -36,6 +42,7 @@ module.exports = (HealthCheckManager = class HealthCheckManager { // keep a record of these objects to dispatch on CHANNEL_MANAGER[this.channel] = this; } + processEvent(id) { // if this is our event record it if (id === this.id) { @@ -46,6 +53,7 @@ module.exports = (HealthCheckManager = class HealthCheckManager { return this.timer = null; // only time the latency of the first event } } + setStatus() { // if we saw the event anything other than a single time that is an error if (this.count !== 1) { @@ -60,13 +68,15 @@ module.exports = (HealthCheckManager = class HealthCheckManager { // dispatch event to manager for channel return (CHANNEL_MANAGER[channel] != null ? CHANNEL_MANAGER[channel].processEvent(id) : undefined); } + static status() { // return status of all channels for logging return CHANNEL_ERROR; } + static isFailing() { // check if any channel status is bad - for (let channel in CHANNEL_ERROR) { + for (const channel in CHANNEL_ERROR) { const error = CHANNEL_ERROR[channel]; if (error === true) { return true; } } diff --git a/services/real-time/app/coffee/HttpApiController.js b/services/real-time/app/coffee/HttpApiController.js index 21a9f15628..88bbc1a5e3 100644 --- a/services/real-time/app/coffee/HttpApiController.js +++ b/services/real-time/app/coffee/HttpApiController.js @@ -1,3 +1,9 @@ +/* eslint-disable + camelcase, + no-unused-vars, +*/ +// TODO: This file was created by bulk-decaffeinate. +// Fix any style issues and re-enable lint. /* * decaffeinate suggestions: * DS101: Remove unnecessary use of Array.from @@ -13,7 +19,7 @@ module.exports = (HttpApiController = { sendMessage(req, res, next) { logger.log({message: req.params.message}, "sending message"); if (Array.isArray(req.body)) { - for (let payload of Array.from(req.body)) { + for (const payload of Array.from(req.body)) { WebsocketLoadBalancer.emitToRoom(req.params.project_id, req.params.message, payload); } } else { diff --git a/services/real-time/app/coffee/HttpController.js b/services/real-time/app/coffee/HttpController.js index aa17c6f6d8..4d33af44b3 100644 --- a/services/real-time/app/coffee/HttpController.js +++ b/services/real-time/app/coffee/HttpController.js @@ -1,3 +1,9 @@ +/* eslint-disable + camelcase, + handle-callback-err, +*/ +// TODO: This file was created by bulk-decaffeinate. +// Fix any style issues and re-enable lint. /* * decaffeinate suggestions: * DS102: Remove unnecessary code created because of implicit returns @@ -19,7 +25,7 @@ module.exports = (HttpController = { const {project_id, user_id, first_name, last_name, email, connected_time} = ioClient.ol_context; const client = {client_id, project_id, user_id, first_name, last_name, email, connected_time}; client.rooms = []; - for (let name in ioClient.manager.roomClients[client_id]) { + for (const name in ioClient.manager.roomClients[client_id]) { const joined = ioClient.manager.roomClients[client_id][name]; if (joined && (name !== "")) { client.rooms.push(name.replace(/^\//, "")); // Remove leading / diff --git a/services/real-time/app/coffee/RedisClientManager.js b/services/real-time/app/coffee/RedisClientManager.js index 7bd33ca914..3da2136b46 100644 --- a/services/real-time/app/coffee/RedisClientManager.js +++ b/services/real-time/app/coffee/RedisClientManager.js @@ -1,3 +1,8 @@ +/* eslint-disable + no-unused-vars, +*/ +// TODO: This file was created by bulk-decaffeinate. +// Fix any style issues and re-enable lint. /* * decaffeinate suggestions: * DS101: Remove unnecessary use of Array.from @@ -14,7 +19,7 @@ module.exports = (RedisClientManager = { // create a dynamic list of redis clients, excluding any configurations which are not defined const clientList = (() => { const result = []; - for (let x of Array.from(configs)) { + for (const x of Array.from(configs)) { if (x != null) { const redisType = (x.cluster != null) ? "cluster" diff --git a/services/real-time/app/coffee/RoomManager.js b/services/real-time/app/coffee/RoomManager.js index c7047e90c0..c75cc68626 100644 --- a/services/real-time/app/coffee/RoomManager.js +++ b/services/real-time/app/coffee/RoomManager.js @@ -1,3 +1,9 @@ +/* eslint-disable + camelcase, + no-unused-vars, +*/ +// TODO: This file was created by bulk-decaffeinate. +// Fix any style issues and re-enable lint. /* * decaffeinate suggestions: * DS101: Remove unnecessary use of Array.from @@ -51,7 +57,7 @@ module.exports = (RoomManager = { logger.log({client: client.id, roomsToLeave}, "client leaving project"); return (() => { const result = []; - for (let id of Array.from(roomsToLeave)) { + for (const id of Array.from(roomsToLeave)) { const entity = IdMap.get(id); result.push(this.leaveEntity(client, entity, id)); } @@ -131,7 +137,7 @@ module.exports = (RoomManager = { _roomsClientIsIn(client) { const roomList = (() => { const result = []; - for (let fullRoomPath in (client.manager.roomClients != null ? client.manager.roomClients[client.id] : undefined)) { + for (const fullRoomPath in (client.manager.roomClients != null ? client.manager.roomClients[client.id] : undefined)) { // strip socket.io prefix from room to get original id if (fullRoomPath !== '') { const [prefix, room] = Array.from(fullRoomPath.split('/', 2)); diff --git a/services/real-time/app/coffee/Router.js b/services/real-time/app/coffee/Router.js index c7ea84192b..f475596036 100644 --- a/services/real-time/app/coffee/Router.js +++ b/services/real-time/app/coffee/Router.js @@ -1,3 +1,10 @@ +/* eslint-disable + camelcase, + handle-callback-err, + standard/no-callback-literal, +*/ +// TODO: This file was created by bulk-decaffeinate. +// Fix any style issues and re-enable lint. /* * decaffeinate suggestions: * DS101: Remove unnecessary use of Array.from @@ -29,7 +36,7 @@ module.exports = (Router = { _handleError(callback, error, client, method, attrs) { if (callback == null) { callback = function(error) {}; } if (attrs == null) { attrs = {}; } - for (let key of ["project_id", "doc_id", "user_id"]) { + for (const key of ["project_id", "doc_id", "user_id"]) { attrs[key] = client.ol_context[key]; } attrs.client_id = client.id; diff --git a/services/real-time/app/coffee/SafeJsonParse.js b/services/real-time/app/coffee/SafeJsonParse.js index 4c058053b7..f5e8dd3797 100644 --- a/services/real-time/app/coffee/SafeJsonParse.js +++ b/services/real-time/app/coffee/SafeJsonParse.js @@ -1,3 +1,8 @@ +/* eslint-disable + handle-callback-err, +*/ +// TODO: This file was created by bulk-decaffeinate. +// Fix any style issues and re-enable lint. /* * decaffeinate suggestions: * DS102: Remove unnecessary code created because of implicit returns diff --git a/services/real-time/app/coffee/SessionSockets.js b/services/real-time/app/coffee/SessionSockets.js index 533fed3d4c..894c7b53d5 100644 --- a/services/real-time/app/coffee/SessionSockets.js +++ b/services/real-time/app/coffee/SessionSockets.js @@ -1,3 +1,5 @@ +// TODO: This file was created by bulk-decaffeinate. +// Sanity-check the conversion and remove this comment. /* * decaffeinate suggestions: * DS102: Remove unnecessary code created because of implicit returns diff --git a/services/real-time/app/coffee/WebApiManager.js b/services/real-time/app/coffee/WebApiManager.js index 489d4c0a7b..9598d83106 100644 --- a/services/real-time/app/coffee/WebApiManager.js +++ b/services/real-time/app/coffee/WebApiManager.js @@ -1,3 +1,10 @@ +/* eslint-disable + camelcase, + handle-callback-err, + no-unused-vars, +*/ +// TODO: This file was created by bulk-decaffeinate. +// Fix any style issues and re-enable lint. /* * decaffeinate suggestions: * DS102: Remove unnecessary code created because of implicit returns @@ -34,7 +41,7 @@ module.exports = (WebApiManager = { }, function(error, response, data) { let err; if (error != null) { return callback(error); } - if (200 <= response.statusCode && response.statusCode < 300) { + if (response.statusCode >= 200 && response.statusCode < 300) { if ((data == null) || ((data != null ? data.project : undefined) == null)) { err = new Error('no data returned from joinProject request'); logger.error({err, project_id, user_id}, "error accessing web api"); diff --git a/services/real-time/app/coffee/WebsocketController.js b/services/real-time/app/coffee/WebsocketController.js index dcd6955ac7..aa51bbb372 100644 --- a/services/real-time/app/coffee/WebsocketController.js +++ b/services/real-time/app/coffee/WebsocketController.js @@ -1,3 +1,10 @@ +/* eslint-disable + camelcase, + handle-callback-err, + no-unused-vars, +*/ +// TODO: This file was created by bulk-decaffeinate. +// Fix any style issues and re-enable lint. /* * decaffeinate suggestions: * DS101: Remove unnecessary use of Array.from @@ -47,17 +54,17 @@ module.exports = (WebsocketController = { } client.ol_context = {}; - client.ol_context["privilege_level"] = privilegeLevel; - client.ol_context["user_id"] = user_id; - client.ol_context["project_id"] = project_id; - client.ol_context["owner_id"] = __guard__(project != null ? project.owner : undefined, x => x._id); - client.ol_context["first_name"] = user != null ? user.first_name : undefined; - client.ol_context["last_name"] = user != null ? user.last_name : undefined; - client.ol_context["email"] = user != null ? user.email : undefined; - client.ol_context["connected_time"] = new Date(); - client.ol_context["signup_date"] = user != null ? user.signUpDate : undefined; - client.ol_context["login_count"] = user != null ? user.loginCount : undefined; - client.ol_context["is_restricted_user"] = !!(isRestrictedUser); + client.ol_context.privilege_level = privilegeLevel; + client.ol_context.user_id = user_id; + client.ol_context.project_id = project_id; + client.ol_context.owner_id = __guard__(project != null ? project.owner : undefined, x => x._id); + client.ol_context.first_name = user != null ? user.first_name : undefined; + client.ol_context.last_name = user != null ? user.last_name : undefined; + client.ol_context.email = user != null ? user.email : undefined; + client.ol_context.connected_time = new Date(); + client.ol_context.signup_date = user != null ? user.signUpDate : undefined; + client.ol_context.login_count = user != null ? user.loginCount : undefined; + client.ol_context.is_restricted_user = !!(isRestrictedUser); RoomManager.joinProject(client, project_id, function(err) { if (err) { return callback(err); } @@ -73,7 +80,7 @@ module.exports = (WebsocketController = { // We want to flush a project if there are no more (local) connected clients // but we need to wait for the triggering client to disconnect. How long we wait // is determined by FLUSH_IF_EMPTY_DELAY. - FLUSH_IF_EMPTY_DELAY: 500, //ms + FLUSH_IF_EMPTY_DELAY: 500, // ms leaveProject(io, client, callback) { if (callback == null) { callback = function(error) {}; } const {project_id, user_id} = client.ol_context; @@ -160,10 +167,10 @@ module.exports = (WebsocketController = { } if (options.encodeRanges) { try { - for (let comment of Array.from((ranges != null ? ranges.comments : undefined) || [])) { + for (const comment of Array.from((ranges != null ? ranges.comments : undefined) || [])) { if (comment.op.c != null) { comment.op.c = encodeForWebsockets(comment.op.c); } } - for (let change of Array.from((ranges != null ? ranges.changes : undefined) || [])) { + for (const change of Array.from((ranges != null ? ranges.changes : undefined) || [])) { if (change.op.i != null) { change.op.i = encodeForWebsockets(change.op.i); } if (change.op.d != null) { change.op.d = encodeForWebsockets(change.op.d); } } @@ -192,7 +199,7 @@ module.exports = (WebsocketController = { // we could remove permission when user leaves a doc, but because // the connection is per-project, we continue to allow access // after the initial joinDoc since we know they are already authorised. - //# AuthorizationManager.removeAccessToDoc client, doc_id + // # AuthorizationManager.removeAccessToDoc client, doc_id return callback(); }, updateClientPosition(client, cursorData, callback) { @@ -221,12 +228,7 @@ module.exports = (WebsocketController = { } else { cursorData.name = first_name && last_name ? `${first_name} ${last_name}` - : first_name ? - first_name - : last_name ? - last_name - : - ""; + : first_name || (last_name || ""); ConnectedUsersManager.updateUserPosition(project_id, client.publicId, { first_name, last_name, @@ -340,7 +342,7 @@ module.exports = (WebsocketController = { }, _isCommentUpdate(update) { - for (let op of Array.from(update.op)) { + for (const op of Array.from(update.op)) { if ((op.c == null)) { return false; } diff --git a/services/real-time/app/coffee/WebsocketLoadBalancer.js b/services/real-time/app/coffee/WebsocketLoadBalancer.js index 0734929453..dc2617742a 100644 --- a/services/real-time/app/coffee/WebsocketLoadBalancer.js +++ b/services/real-time/app/coffee/WebsocketLoadBalancer.js @@ -1,3 +1,8 @@ +/* eslint-disable + camelcase, +*/ +// TODO: This file was created by bulk-decaffeinate. +// Fix any style issues and re-enable lint. /* * decaffeinate suggestions: * DS101: Remove unnecessary use of Array.from @@ -55,7 +60,7 @@ module.exports = (WebsocketLoadBalancer = { listenForEditorEvents(io) { logger.log({rclients: this.rclientPubList.length}, "publishing editor events"); logger.log({rclients: this.rclientSubList.length}, "listening for editor events"); - for (let rclientSub of Array.from(this.rclientSubList)) { + for (const rclientSub of Array.from(this.rclientSubList)) { rclientSub.subscribe("editor-events"); rclientSub.on("message", function(channel, message) { if (Settings.debugEvents > 0) { EventLogger.debugEvent(channel, message); } @@ -113,7 +118,7 @@ module.exports = (WebsocketLoadBalancer = { // send messages only to unique clients (due to duplicate entries in io.sockets.clients) clientList = io.sockets.clients(message.room_id) - .filter(client => !(is_restricted_message && client.ol_context['is_restricted_user'])); + .filter(client => !(is_restricted_message && client.ol_context.is_restricted_user)); // avoid unnecessary work if no clients are connected if (clientList.length === 0) { return; } From 04a85a67162f5de717872006a21f11ee6743909f Mon Sep 17 00:00:00 2001 From: decaffeinate Date: Tue, 23 Jun 2020 18:29:41 +0100 Subject: [PATCH 371/491] decaffeinate: rename app/coffee dir to app/js --- services/real-time/app/{coffee => js}/AuthorizationManager.js | 0 services/real-time/app/{coffee => js}/ChannelManager.js | 0 services/real-time/app/{coffee => js}/ConnectedUsersManager.js | 0 .../real-time/app/{coffee => js}/DocumentUpdaterController.js | 0 services/real-time/app/{coffee => js}/DocumentUpdaterManager.js | 0 services/real-time/app/{coffee => js}/DrainManager.js | 0 services/real-time/app/{coffee => js}/Errors.js | 0 services/real-time/app/{coffee => js}/EventLogger.js | 0 services/real-time/app/{coffee => js}/HealthCheckManager.js | 0 services/real-time/app/{coffee => js}/HttpApiController.js | 0 services/real-time/app/{coffee => js}/HttpController.js | 0 services/real-time/app/{coffee => js}/RedisClientManager.js | 0 services/real-time/app/{coffee => js}/RoomManager.js | 0 services/real-time/app/{coffee => js}/Router.js | 0 services/real-time/app/{coffee => js}/SafeJsonParse.js | 0 services/real-time/app/{coffee => js}/SessionSockets.js | 0 services/real-time/app/{coffee => js}/WebApiManager.js | 0 services/real-time/app/{coffee => js}/WebsocketController.js | 0 services/real-time/app/{coffee => js}/WebsocketLoadBalancer.js | 0 19 files changed, 0 insertions(+), 0 deletions(-) rename services/real-time/app/{coffee => js}/AuthorizationManager.js (100%) rename services/real-time/app/{coffee => js}/ChannelManager.js (100%) rename services/real-time/app/{coffee => js}/ConnectedUsersManager.js (100%) rename services/real-time/app/{coffee => js}/DocumentUpdaterController.js (100%) rename services/real-time/app/{coffee => js}/DocumentUpdaterManager.js (100%) rename services/real-time/app/{coffee => js}/DrainManager.js (100%) rename services/real-time/app/{coffee => js}/Errors.js (100%) rename services/real-time/app/{coffee => js}/EventLogger.js (100%) rename services/real-time/app/{coffee => js}/HealthCheckManager.js (100%) rename services/real-time/app/{coffee => js}/HttpApiController.js (100%) rename services/real-time/app/{coffee => js}/HttpController.js (100%) rename services/real-time/app/{coffee => js}/RedisClientManager.js (100%) rename services/real-time/app/{coffee => js}/RoomManager.js (100%) rename services/real-time/app/{coffee => js}/Router.js (100%) rename services/real-time/app/{coffee => js}/SafeJsonParse.js (100%) rename services/real-time/app/{coffee => js}/SessionSockets.js (100%) rename services/real-time/app/{coffee => js}/WebApiManager.js (100%) rename services/real-time/app/{coffee => js}/WebsocketController.js (100%) rename services/real-time/app/{coffee => js}/WebsocketLoadBalancer.js (100%) diff --git a/services/real-time/app/coffee/AuthorizationManager.js b/services/real-time/app/js/AuthorizationManager.js similarity index 100% rename from services/real-time/app/coffee/AuthorizationManager.js rename to services/real-time/app/js/AuthorizationManager.js diff --git a/services/real-time/app/coffee/ChannelManager.js b/services/real-time/app/js/ChannelManager.js similarity index 100% rename from services/real-time/app/coffee/ChannelManager.js rename to services/real-time/app/js/ChannelManager.js diff --git a/services/real-time/app/coffee/ConnectedUsersManager.js b/services/real-time/app/js/ConnectedUsersManager.js similarity index 100% rename from services/real-time/app/coffee/ConnectedUsersManager.js rename to services/real-time/app/js/ConnectedUsersManager.js diff --git a/services/real-time/app/coffee/DocumentUpdaterController.js b/services/real-time/app/js/DocumentUpdaterController.js similarity index 100% rename from services/real-time/app/coffee/DocumentUpdaterController.js rename to services/real-time/app/js/DocumentUpdaterController.js diff --git a/services/real-time/app/coffee/DocumentUpdaterManager.js b/services/real-time/app/js/DocumentUpdaterManager.js similarity index 100% rename from services/real-time/app/coffee/DocumentUpdaterManager.js rename to services/real-time/app/js/DocumentUpdaterManager.js diff --git a/services/real-time/app/coffee/DrainManager.js b/services/real-time/app/js/DrainManager.js similarity index 100% rename from services/real-time/app/coffee/DrainManager.js rename to services/real-time/app/js/DrainManager.js diff --git a/services/real-time/app/coffee/Errors.js b/services/real-time/app/js/Errors.js similarity index 100% rename from services/real-time/app/coffee/Errors.js rename to services/real-time/app/js/Errors.js diff --git a/services/real-time/app/coffee/EventLogger.js b/services/real-time/app/js/EventLogger.js similarity index 100% rename from services/real-time/app/coffee/EventLogger.js rename to services/real-time/app/js/EventLogger.js diff --git a/services/real-time/app/coffee/HealthCheckManager.js b/services/real-time/app/js/HealthCheckManager.js similarity index 100% rename from services/real-time/app/coffee/HealthCheckManager.js rename to services/real-time/app/js/HealthCheckManager.js diff --git a/services/real-time/app/coffee/HttpApiController.js b/services/real-time/app/js/HttpApiController.js similarity index 100% rename from services/real-time/app/coffee/HttpApiController.js rename to services/real-time/app/js/HttpApiController.js diff --git a/services/real-time/app/coffee/HttpController.js b/services/real-time/app/js/HttpController.js similarity index 100% rename from services/real-time/app/coffee/HttpController.js rename to services/real-time/app/js/HttpController.js diff --git a/services/real-time/app/coffee/RedisClientManager.js b/services/real-time/app/js/RedisClientManager.js similarity index 100% rename from services/real-time/app/coffee/RedisClientManager.js rename to services/real-time/app/js/RedisClientManager.js diff --git a/services/real-time/app/coffee/RoomManager.js b/services/real-time/app/js/RoomManager.js similarity index 100% rename from services/real-time/app/coffee/RoomManager.js rename to services/real-time/app/js/RoomManager.js diff --git a/services/real-time/app/coffee/Router.js b/services/real-time/app/js/Router.js similarity index 100% rename from services/real-time/app/coffee/Router.js rename to services/real-time/app/js/Router.js diff --git a/services/real-time/app/coffee/SafeJsonParse.js b/services/real-time/app/js/SafeJsonParse.js similarity index 100% rename from services/real-time/app/coffee/SafeJsonParse.js rename to services/real-time/app/js/SafeJsonParse.js diff --git a/services/real-time/app/coffee/SessionSockets.js b/services/real-time/app/js/SessionSockets.js similarity index 100% rename from services/real-time/app/coffee/SessionSockets.js rename to services/real-time/app/js/SessionSockets.js diff --git a/services/real-time/app/coffee/WebApiManager.js b/services/real-time/app/js/WebApiManager.js similarity index 100% rename from services/real-time/app/coffee/WebApiManager.js rename to services/real-time/app/js/WebApiManager.js diff --git a/services/real-time/app/coffee/WebsocketController.js b/services/real-time/app/js/WebsocketController.js similarity index 100% rename from services/real-time/app/coffee/WebsocketController.js rename to services/real-time/app/js/WebsocketController.js diff --git a/services/real-time/app/coffee/WebsocketLoadBalancer.js b/services/real-time/app/js/WebsocketLoadBalancer.js similarity index 100% rename from services/real-time/app/coffee/WebsocketLoadBalancer.js rename to services/real-time/app/js/WebsocketLoadBalancer.js From 817844515dcc0432755c50a01153ce129cdcb5f8 Mon Sep 17 00:00:00 2001 From: decaffeinate Date: Tue, 23 Jun 2020 18:29:44 +0100 Subject: [PATCH 372/491] prettier: convert app/js decaffeinated files to Prettier format --- .../real-time/app/js/AuthorizationManager.js | 144 +-- services/real-time/app/js/ChannelManager.js | 158 ++-- .../real-time/app/js/ConnectedUsersManager.js | 269 ++++-- .../app/js/DocumentUpdaterController.js | 308 ++++--- .../app/js/DocumentUpdaterManager.js | 243 +++-- services/real-time/app/js/DrainManager.js | 97 +- services/real-time/app/js/Errors.js | 21 +- services/real-time/app/js/EventLogger.js | 147 +-- .../real-time/app/js/HealthCheckManager.js | 130 +-- .../real-time/app/js/HttpApiController.js | 88 +- services/real-time/app/js/HttpController.js | 118 ++- .../real-time/app/js/RedisClientManager.js | 54 +- services/real-time/app/js/RoomManager.js | 291 +++--- services/real-time/app/js/Router.js | 583 +++++++----- services/real-time/app/js/SafeJsonParse.js | 39 +- services/real-time/app/js/SessionSockets.js | 51 +- services/real-time/app/js/WebApiManager.js | 119 ++- .../real-time/app/js/WebsocketController.js | 866 +++++++++++------- .../real-time/app/js/WebsocketLoadBalancer.js | 323 ++++--- 19 files changed, 2425 insertions(+), 1624 deletions(-) diff --git a/services/real-time/app/js/AuthorizationManager.js b/services/real-time/app/js/AuthorizationManager.js index 41caee9ef2..15607a898a 100644 --- a/services/real-time/app/js/AuthorizationManager.js +++ b/services/real-time/app/js/AuthorizationManager.js @@ -11,61 +11,101 @@ * DS207: Consider shorter variations of null checks * Full docs: https://github.com/decaffeinate/decaffeinate/blob/master/docs/suggestions.md */ -let AuthorizationManager; -module.exports = (AuthorizationManager = { - assertClientCanViewProject(client, callback) { - if (callback == null) { callback = function(error) {}; } - return AuthorizationManager._assertClientHasPrivilegeLevel(client, ["readOnly", "readAndWrite", "owner"], callback); - }, +let AuthorizationManager +module.exports = AuthorizationManager = { + assertClientCanViewProject(client, callback) { + if (callback == null) { + callback = function (error) {} + } + return AuthorizationManager._assertClientHasPrivilegeLevel( + client, + ['readOnly', 'readAndWrite', 'owner'], + callback + ) + }, - assertClientCanEditProject(client, callback) { - if (callback == null) { callback = function(error) {}; } - return AuthorizationManager._assertClientHasPrivilegeLevel(client, ["readAndWrite", "owner"], callback); - }, - - _assertClientHasPrivilegeLevel(client, allowedLevels, callback) { - if (callback == null) { callback = function(error) {}; } - if (Array.from(allowedLevels).includes(client.ol_context.privilege_level)) { - return callback(null); - } else { - return callback(new Error("not authorized")); - } - }, + assertClientCanEditProject(client, callback) { + if (callback == null) { + callback = function (error) {} + } + return AuthorizationManager._assertClientHasPrivilegeLevel( + client, + ['readAndWrite', 'owner'], + callback + ) + }, - assertClientCanViewProjectAndDoc(client, doc_id, callback) { - if (callback == null) { callback = function(error) {}; } - return AuthorizationManager.assertClientCanViewProject(client, function(error) { - if (error != null) { return callback(error); } - return AuthorizationManager._assertClientCanAccessDoc(client, doc_id, callback); - }); - }, + _assertClientHasPrivilegeLevel(client, allowedLevels, callback) { + if (callback == null) { + callback = function (error) {} + } + if (Array.from(allowedLevels).includes(client.ol_context.privilege_level)) { + return callback(null) + } else { + return callback(new Error('not authorized')) + } + }, - assertClientCanEditProjectAndDoc(client, doc_id, callback) { - if (callback == null) { callback = function(error) {}; } - return AuthorizationManager.assertClientCanEditProject(client, function(error) { - if (error != null) { return callback(error); } - return AuthorizationManager._assertClientCanAccessDoc(client, doc_id, callback); - }); - }, + assertClientCanViewProjectAndDoc(client, doc_id, callback) { + if (callback == null) { + callback = function (error) {} + } + return AuthorizationManager.assertClientCanViewProject(client, function ( + error + ) { + if (error != null) { + return callback(error) + } + return AuthorizationManager._assertClientCanAccessDoc( + client, + doc_id, + callback + ) + }) + }, - _assertClientCanAccessDoc(client, doc_id, callback) { - if (callback == null) { callback = function(error) {}; } - if (client.ol_context[`doc:${doc_id}`] === "allowed") { - return callback(null); - } else { - return callback(new Error("not authorized")); - } - }, + assertClientCanEditProjectAndDoc(client, doc_id, callback) { + if (callback == null) { + callback = function (error) {} + } + return AuthorizationManager.assertClientCanEditProject(client, function ( + error + ) { + if (error != null) { + return callback(error) + } + return AuthorizationManager._assertClientCanAccessDoc( + client, + doc_id, + callback + ) + }) + }, - addAccessToDoc(client, doc_id, callback) { - if (callback == null) { callback = function(error) {}; } - client.ol_context[`doc:${doc_id}`] = "allowed"; - return callback(null); - }, + _assertClientCanAccessDoc(client, doc_id, callback) { + if (callback == null) { + callback = function (error) {} + } + if (client.ol_context[`doc:${doc_id}`] === 'allowed') { + return callback(null) + } else { + return callback(new Error('not authorized')) + } + }, - removeAccessToDoc(client, doc_id, callback) { - if (callback == null) { callback = function(error) {}; } - delete client.ol_context[`doc:${doc_id}`]; - return callback(null); - } -}); + addAccessToDoc(client, doc_id, callback) { + if (callback == null) { + callback = function (error) {} + } + client.ol_context[`doc:${doc_id}`] = 'allowed' + return callback(null) + }, + + removeAccessToDoc(client, doc_id, callback) { + if (callback == null) { + callback = function (error) {} + } + delete client.ol_context[`doc:${doc_id}`] + return callback(null) + } +} diff --git a/services/real-time/app/js/ChannelManager.js b/services/real-time/app/js/ChannelManager.js index 60e7c5c635..09e81cebf5 100644 --- a/services/real-time/app/js/ChannelManager.js +++ b/services/real-time/app/js/ChannelManager.js @@ -8,84 +8,98 @@ * DS102: Remove unnecessary code created because of implicit returns * Full docs: https://github.com/decaffeinate/decaffeinate/blob/master/docs/suggestions.md */ -let ChannelManager; -const logger = require('logger-sharelatex'); -const metrics = require("metrics-sharelatex"); -const settings = require("settings-sharelatex"); +let ChannelManager +const logger = require('logger-sharelatex') +const metrics = require('metrics-sharelatex') +const settings = require('settings-sharelatex') -const ClientMap = new Map(); // for each redis client, store a Map of subscribed channels (channelname -> subscribe promise) +const ClientMap = new Map() // for each redis client, store a Map of subscribed channels (channelname -> subscribe promise) // Manage redis pubsub subscriptions for individual projects and docs, ensuring // that we never subscribe to a channel multiple times. The socket.io side is // handled by RoomManager. -module.exports = (ChannelManager = { - getClientMapEntry(rclient) { - // return the per-client channel map if it exists, otherwise create and - // return an empty map for the client. - return ClientMap.get(rclient) || ClientMap.set(rclient, new Map()).get(rclient); - }, +module.exports = ChannelManager = { + getClientMapEntry(rclient) { + // return the per-client channel map if it exists, otherwise create and + // return an empty map for the client. + return ( + ClientMap.get(rclient) || ClientMap.set(rclient, new Map()).get(rclient) + ) + }, - subscribe(rclient, baseChannel, id) { - const clientChannelMap = this.getClientMapEntry(rclient); - const channel = `${baseChannel}:${id}`; - const actualSubscribe = function() { - // subscribe is happening in the foreground and it should reject - const p = rclient.subscribe(channel); - p.finally(function() { - if (clientChannelMap.get(channel) === subscribePromise) { - return clientChannelMap.delete(channel); - }}).then(function() { - logger.log({channel}, "subscribed to channel"); - return metrics.inc(`subscribe.${baseChannel}`);}).catch(function(err) { - logger.error({channel, err}, "failed to subscribe to channel"); - return metrics.inc(`subscribe.failed.${baseChannel}`); - }); - return p; - }; - - const pendingActions = clientChannelMap.get(channel) || Promise.resolve(); - var subscribePromise = pendingActions.then(actualSubscribe, actualSubscribe); - clientChannelMap.set(channel, subscribePromise); - logger.log({channel}, "planned to subscribe to channel"); - return subscribePromise; - }, - - unsubscribe(rclient, baseChannel, id) { - const clientChannelMap = this.getClientMapEntry(rclient); - const channel = `${baseChannel}:${id}`; - const actualUnsubscribe = function() { - // unsubscribe is happening in the background, it should not reject - const p = rclient.unsubscribe(channel) - .finally(function() { - if (clientChannelMap.get(channel) === unsubscribePromise) { - return clientChannelMap.delete(channel); - }}).then(function() { - logger.log({channel}, "unsubscribed from channel"); - return metrics.inc(`unsubscribe.${baseChannel}`);}).catch(function(err) { - logger.error({channel, err}, "unsubscribed from channel"); - return metrics.inc(`unsubscribe.failed.${baseChannel}`); - }); - return p; - }; - - const pendingActions = clientChannelMap.get(channel) || Promise.resolve(); - var unsubscribePromise = pendingActions.then(actualUnsubscribe, actualUnsubscribe); - clientChannelMap.set(channel, unsubscribePromise); - logger.log({channel}, "planned to unsubscribe from channel"); - return unsubscribePromise; - }, - - publish(rclient, baseChannel, id, data) { - let channel; - metrics.summary(`redis.publish.${baseChannel}`, data.length); - if ((id === 'all') || !settings.publishOnIndividualChannels) { - channel = baseChannel; - } else { - channel = `${baseChannel}:${id}`; + subscribe(rclient, baseChannel, id) { + const clientChannelMap = this.getClientMapEntry(rclient) + const channel = `${baseChannel}:${id}` + const actualSubscribe = function () { + // subscribe is happening in the foreground and it should reject + const p = rclient.subscribe(channel) + p.finally(function () { + if (clientChannelMap.get(channel) === subscribePromise) { + return clientChannelMap.delete(channel) } - // we publish on a different client to the subscribe, so we can't - // check for the channel existing here - return rclient.publish(channel, data); + }) + .then(function () { + logger.log({ channel }, 'subscribed to channel') + return metrics.inc(`subscribe.${baseChannel}`) + }) + .catch(function (err) { + logger.error({ channel, err }, 'failed to subscribe to channel') + return metrics.inc(`subscribe.failed.${baseChannel}`) + }) + return p } -}); + + const pendingActions = clientChannelMap.get(channel) || Promise.resolve() + var subscribePromise = pendingActions.then(actualSubscribe, actualSubscribe) + clientChannelMap.set(channel, subscribePromise) + logger.log({ channel }, 'planned to subscribe to channel') + return subscribePromise + }, + + unsubscribe(rclient, baseChannel, id) { + const clientChannelMap = this.getClientMapEntry(rclient) + const channel = `${baseChannel}:${id}` + const actualUnsubscribe = function () { + // unsubscribe is happening in the background, it should not reject + const p = rclient + .unsubscribe(channel) + .finally(function () { + if (clientChannelMap.get(channel) === unsubscribePromise) { + return clientChannelMap.delete(channel) + } + }) + .then(function () { + logger.log({ channel }, 'unsubscribed from channel') + return metrics.inc(`unsubscribe.${baseChannel}`) + }) + .catch(function (err) { + logger.error({ channel, err }, 'unsubscribed from channel') + return metrics.inc(`unsubscribe.failed.${baseChannel}`) + }) + return p + } + + const pendingActions = clientChannelMap.get(channel) || Promise.resolve() + var unsubscribePromise = pendingActions.then( + actualUnsubscribe, + actualUnsubscribe + ) + clientChannelMap.set(channel, unsubscribePromise) + logger.log({ channel }, 'planned to unsubscribe from channel') + return unsubscribePromise + }, + + publish(rclient, baseChannel, id, data) { + let channel + metrics.summary(`redis.publish.${baseChannel}`, data.length) + if (id === 'all' || !settings.publishOnIndividualChannels) { + channel = baseChannel + } else { + channel = `${baseChannel}:${id}` + } + // we publish on a different client to the subscribe, so we can't + // check for the channel existing here + return rclient.publish(channel, data) + } +} diff --git a/services/real-time/app/js/ConnectedUsersManager.js b/services/real-time/app/js/ConnectedUsersManager.js index b2a0fc2eee..6770dd5421 100644 --- a/services/real-time/app/js/ConnectedUsersManager.js +++ b/services/real-time/app/js/ConnectedUsersManager.js @@ -10,112 +10,185 @@ * DS207: Consider shorter variations of null checks * Full docs: https://github.com/decaffeinate/decaffeinate/blob/master/docs/suggestions.md */ -const async = require("async"); -const Settings = require('settings-sharelatex'); -const logger = require("logger-sharelatex"); -const redis = require("redis-sharelatex"); -const rclient = redis.createClient(Settings.redis.realtime); -const Keys = Settings.redis.realtime.key_schema; +const async = require('async') +const Settings = require('settings-sharelatex') +const logger = require('logger-sharelatex') +const redis = require('redis-sharelatex') +const rclient = redis.createClient(Settings.redis.realtime) +const Keys = Settings.redis.realtime.key_schema -const ONE_HOUR_IN_S = 60 * 60; -const ONE_DAY_IN_S = ONE_HOUR_IN_S * 24; -const FOUR_DAYS_IN_S = ONE_DAY_IN_S * 4; +const ONE_HOUR_IN_S = 60 * 60 +const ONE_DAY_IN_S = ONE_HOUR_IN_S * 24 +const FOUR_DAYS_IN_S = ONE_DAY_IN_S * 4 -const USER_TIMEOUT_IN_S = ONE_HOUR_IN_S / 4; -const REFRESH_TIMEOUT_IN_S = 10; // only show clients which have responded to a refresh request in the last 10 seconds +const USER_TIMEOUT_IN_S = ONE_HOUR_IN_S / 4 +const REFRESH_TIMEOUT_IN_S = 10 // only show clients which have responded to a refresh request in the last 10 seconds module.exports = { - // Use the same method for when a user connects, and when a user sends a cursor - // update. This way we don't care if the connected_user key has expired when - // we receive a cursor update. - updateUserPosition(project_id, client_id, user, cursorData, callback){ - if (callback == null) { callback = function(err){}; } - logger.log({project_id, client_id}, "marking user as joined or connected"); + // Use the same method for when a user connects, and when a user sends a cursor + // update. This way we don't care if the connected_user key has expired when + // we receive a cursor update. + updateUserPosition(project_id, client_id, user, cursorData, callback) { + if (callback == null) { + callback = function (err) {} + } + logger.log({ project_id, client_id }, 'marking user as joined or connected') - const multi = rclient.multi(); - - multi.sadd(Keys.clientsInProject({project_id}), client_id); - multi.expire(Keys.clientsInProject({project_id}), FOUR_DAYS_IN_S); - - multi.hset(Keys.connectedUser({project_id, client_id}), "last_updated_at", Date.now()); - multi.hset(Keys.connectedUser({project_id, client_id}), "user_id", user._id); - multi.hset(Keys.connectedUser({project_id, client_id}), "first_name", user.first_name || ""); - multi.hset(Keys.connectedUser({project_id, client_id}), "last_name", user.last_name || ""); - multi.hset(Keys.connectedUser({project_id, client_id}), "email", user.email || ""); - - if (cursorData != null) { - multi.hset(Keys.connectedUser({project_id, client_id}), "cursorData", JSON.stringify(cursorData)); - } - multi.expire(Keys.connectedUser({project_id, client_id}), USER_TIMEOUT_IN_S); - - return multi.exec(function(err){ - if (err != null) { - logger.err({err, project_id, client_id}, "problem marking user as connected"); - } - return callback(err); - }); - }, + const multi = rclient.multi() - refreshClient(project_id, client_id, callback) { - if (callback == null) { callback = function(err) {}; } - logger.log({project_id, client_id}, "refreshing connected client"); - const multi = rclient.multi(); - multi.hset(Keys.connectedUser({project_id, client_id}), "last_updated_at", Date.now()); - multi.expire(Keys.connectedUser({project_id, client_id}), USER_TIMEOUT_IN_S); - return multi.exec(function(err){ - if (err != null) { - logger.err({err, project_id, client_id}, "problem refreshing connected client"); - } - return callback(err); - }); - }, + multi.sadd(Keys.clientsInProject({ project_id }), client_id) + multi.expire(Keys.clientsInProject({ project_id }), FOUR_DAYS_IN_S) - markUserAsDisconnected(project_id, client_id, callback){ - logger.log({project_id, client_id}, "marking user as disconnected"); - const multi = rclient.multi(); - multi.srem(Keys.clientsInProject({project_id}), client_id); - multi.expire(Keys.clientsInProject({project_id}), FOUR_DAYS_IN_S); - multi.del(Keys.connectedUser({project_id, client_id})); - return multi.exec(callback); - }, + multi.hset( + Keys.connectedUser({ project_id, client_id }), + 'last_updated_at', + Date.now() + ) + multi.hset( + Keys.connectedUser({ project_id, client_id }), + 'user_id', + user._id + ) + multi.hset( + Keys.connectedUser({ project_id, client_id }), + 'first_name', + user.first_name || '' + ) + multi.hset( + Keys.connectedUser({ project_id, client_id }), + 'last_name', + user.last_name || '' + ) + multi.hset( + Keys.connectedUser({ project_id, client_id }), + 'email', + user.email || '' + ) + if (cursorData != null) { + multi.hset( + Keys.connectedUser({ project_id, client_id }), + 'cursorData', + JSON.stringify(cursorData) + ) + } + multi.expire( + Keys.connectedUser({ project_id, client_id }), + USER_TIMEOUT_IN_S + ) - _getConnectedUser(project_id, client_id, callback){ - return rclient.hgetall(Keys.connectedUser({project_id, client_id}), function(err, result){ - if ((result == null) || (Object.keys(result).length === 0) || !result.user_id) { - result = { - connected : false, - client_id - }; - } else { - result.connected = true; - result.client_id = client_id; - result.client_age = (Date.now() - parseInt(result.last_updated_at,10)) / 1000; - if (result.cursorData != null) { - try { - result.cursorData = JSON.parse(result.cursorData); - } catch (e) { - logger.error({err: e, project_id, client_id, cursorData: result.cursorData}, "error parsing cursorData JSON"); - return callback(e); - } - } - } - return callback(err, result); - }); - }, + return multi.exec(function (err) { + if (err != null) { + logger.err( + { err, project_id, client_id }, + 'problem marking user as connected' + ) + } + return callback(err) + }) + }, - getConnectedUsers(project_id, callback){ - const self = this; - return rclient.smembers(Keys.clientsInProject({project_id}), function(err, results){ - if (err != null) { return callback(err); } - const jobs = results.map(client_id => cb => self._getConnectedUser(project_id, client_id, cb)); - return async.series(jobs, function(err, users){ - if (users == null) { users = []; } - if (err != null) { return callback(err); } - users = users.filter(user => (user != null ? user.connected : undefined) && ((user != null ? user.client_age : undefined) < REFRESH_TIMEOUT_IN_S)); - return callback(null, users); - }); - }); - } -}; + refreshClient(project_id, client_id, callback) { + if (callback == null) { + callback = function (err) {} + } + logger.log({ project_id, client_id }, 'refreshing connected client') + const multi = rclient.multi() + multi.hset( + Keys.connectedUser({ project_id, client_id }), + 'last_updated_at', + Date.now() + ) + multi.expire( + Keys.connectedUser({ project_id, client_id }), + USER_TIMEOUT_IN_S + ) + return multi.exec(function (err) { + if (err != null) { + logger.err( + { err, project_id, client_id }, + 'problem refreshing connected client' + ) + } + return callback(err) + }) + }, + markUserAsDisconnected(project_id, client_id, callback) { + logger.log({ project_id, client_id }, 'marking user as disconnected') + const multi = rclient.multi() + multi.srem(Keys.clientsInProject({ project_id }), client_id) + multi.expire(Keys.clientsInProject({ project_id }), FOUR_DAYS_IN_S) + multi.del(Keys.connectedUser({ project_id, client_id })) + return multi.exec(callback) + }, + + _getConnectedUser(project_id, client_id, callback) { + return rclient.hgetall( + Keys.connectedUser({ project_id, client_id }), + function (err, result) { + if ( + result == null || + Object.keys(result).length === 0 || + !result.user_id + ) { + result = { + connected: false, + client_id + } + } else { + result.connected = true + result.client_id = client_id + result.client_age = + (Date.now() - parseInt(result.last_updated_at, 10)) / 1000 + if (result.cursorData != null) { + try { + result.cursorData = JSON.parse(result.cursorData) + } catch (e) { + logger.error( + { + err: e, + project_id, + client_id, + cursorData: result.cursorData + }, + 'error parsing cursorData JSON' + ) + return callback(e) + } + } + } + return callback(err, result) + } + ) + }, + + getConnectedUsers(project_id, callback) { + const self = this + return rclient.smembers(Keys.clientsInProject({ project_id }), function ( + err, + results + ) { + if (err != null) { + return callback(err) + } + const jobs = results.map((client_id) => (cb) => + self._getConnectedUser(project_id, client_id, cb) + ) + return async.series(jobs, function (err, users) { + if (users == null) { + users = [] + } + if (err != null) { + return callback(err) + } + users = users.filter( + (user) => + (user != null ? user.connected : undefined) && + (user != null ? user.client_age : undefined) < REFRESH_TIMEOUT_IN_S + ) + return callback(null, users) + }) + }) + } +} diff --git a/services/real-time/app/js/DocumentUpdaterController.js b/services/real-time/app/js/DocumentUpdaterController.js index cbf5c600fd..b8dde3b426 100644 --- a/services/real-time/app/js/DocumentUpdaterController.js +++ b/services/real-time/app/js/DocumentUpdaterController.js @@ -12,131 +12,197 @@ * DS207: Consider shorter variations of null checks * Full docs: https://github.com/decaffeinate/decaffeinate/blob/master/docs/suggestions.md */ -let DocumentUpdaterController; -const logger = require("logger-sharelatex"); -const settings = require('settings-sharelatex'); -const RedisClientManager = require("./RedisClientManager"); -const SafeJsonParse = require("./SafeJsonParse"); -const EventLogger = require("./EventLogger"); -const HealthCheckManager = require("./HealthCheckManager"); -const RoomManager = require("./RoomManager"); -const ChannelManager = require("./ChannelManager"); -const metrics = require("metrics-sharelatex"); +let DocumentUpdaterController +const logger = require('logger-sharelatex') +const settings = require('settings-sharelatex') +const RedisClientManager = require('./RedisClientManager') +const SafeJsonParse = require('./SafeJsonParse') +const EventLogger = require('./EventLogger') +const HealthCheckManager = require('./HealthCheckManager') +const RoomManager = require('./RoomManager') +const ChannelManager = require('./ChannelManager') +const metrics = require('metrics-sharelatex') -const MESSAGE_SIZE_LOG_LIMIT = 1024 * 1024; // 1Mb +const MESSAGE_SIZE_LOG_LIMIT = 1024 * 1024 // 1Mb -module.exports = (DocumentUpdaterController = { - // DocumentUpdaterController is responsible for updates that come via Redis - // Pub/Sub from the document updater. - rclientList: RedisClientManager.createClientList(settings.redis.pubsub), +module.exports = DocumentUpdaterController = { + // DocumentUpdaterController is responsible for updates that come via Redis + // Pub/Sub from the document updater. + rclientList: RedisClientManager.createClientList(settings.redis.pubsub), - listenForUpdatesFromDocumentUpdater(io) { - let i, rclient; - logger.log({rclients: this.rclientList.length}, "listening for applied-ops events"); - for (i = 0; i < this.rclientList.length; i++) { - rclient = this.rclientList[i]; - rclient.subscribe("applied-ops"); - rclient.on("message", function(channel, message) { - metrics.inc("rclient", 0.001); // global event rate metric - if (settings.debugEvents > 0) { EventLogger.debugEvent(channel, message); } - return DocumentUpdaterController._processMessageFromDocumentUpdater(io, channel, message); - }); - } - // create metrics for each redis instance only when we have multiple redis clients - if (this.rclientList.length > 1) { - for (i = 0; i < this.rclientList.length; i++) { - rclient = this.rclientList[i]; - ((i => // per client event rate metric - rclient.on("message", () => metrics.inc(`rclient-${i}`, 0.001))))(i); - } - } - return this.handleRoomUpdates(this.rclientList); - }, + listenForUpdatesFromDocumentUpdater(io) { + let i, rclient + logger.log( + { rclients: this.rclientList.length }, + 'listening for applied-ops events' + ) + for (i = 0; i < this.rclientList.length; i++) { + rclient = this.rclientList[i] + rclient.subscribe('applied-ops') + rclient.on('message', function (channel, message) { + metrics.inc('rclient', 0.001) // global event rate metric + if (settings.debugEvents > 0) { + EventLogger.debugEvent(channel, message) + } + return DocumentUpdaterController._processMessageFromDocumentUpdater( + io, + channel, + message + ) + }) + } + // create metrics for each redis instance only when we have multiple redis clients + if (this.rclientList.length > 1) { + for (i = 0; i < this.rclientList.length; i++) { + rclient = this.rclientList[i] + ;(( + i // per client event rate metric + ) => rclient.on('message', () => metrics.inc(`rclient-${i}`, 0.001)))(i) + } + } + return this.handleRoomUpdates(this.rclientList) + }, - handleRoomUpdates(rclientSubList) { - const roomEvents = RoomManager.eventSource(); - roomEvents.on('doc-active', function(doc_id) { - const subscribePromises = Array.from(rclientSubList).map((rclient) => - ChannelManager.subscribe(rclient, "applied-ops", doc_id)); - return RoomManager.emitOnCompletion(subscribePromises, `doc-subscribed-${doc_id}`); - }); - return roomEvents.on('doc-empty', doc_id => Array.from(rclientSubList).map((rclient) => - ChannelManager.unsubscribe(rclient, "applied-ops", doc_id))); - }, + handleRoomUpdates(rclientSubList) { + const roomEvents = RoomManager.eventSource() + roomEvents.on('doc-active', function (doc_id) { + const subscribePromises = Array.from(rclientSubList).map((rclient) => + ChannelManager.subscribe(rclient, 'applied-ops', doc_id) + ) + return RoomManager.emitOnCompletion( + subscribePromises, + `doc-subscribed-${doc_id}` + ) + }) + return roomEvents.on('doc-empty', (doc_id) => + Array.from(rclientSubList).map((rclient) => + ChannelManager.unsubscribe(rclient, 'applied-ops', doc_id) + ) + ) + }, - _processMessageFromDocumentUpdater(io, channel, message) { - return SafeJsonParse.parse(message, function(error, message) { - if (error != null) { - logger.error({err: error, channel}, "error parsing JSON"); - return; - } - if (message.op != null) { - if ((message._id != null) && settings.checkEventOrder) { - const status = EventLogger.checkEventOrder("applied-ops", message._id, message); - if (status === 'duplicate') { - return; // skip duplicate events - } - } - return DocumentUpdaterController._applyUpdateFromDocumentUpdater(io, message.doc_id, message.op); - } else if (message.error != null) { - return DocumentUpdaterController._processErrorFromDocumentUpdater(io, message.doc_id, message.error, message); - } else if (message.health_check != null) { - logger.debug({message}, "got health check message in applied ops channel"); - return HealthCheckManager.check(channel, message.key); - } - }); - }, - - _applyUpdateFromDocumentUpdater(io, doc_id, update) { - let client; - const clientList = io.sockets.clients(doc_id); - // avoid unnecessary work if no clients are connected - if (clientList.length === 0) { - return; - } - // send updates to clients - logger.log({doc_id, version: update.v, source: (update.meta != null ? update.meta.source : undefined), socketIoClients: (((() => { - const result = []; - for (client of Array.from(clientList)) { result.push(client.id); - } - return result; - })()))}, "distributing updates to clients"); - const seen = {}; - // send messages only to unique clients (due to duplicate entries in io.sockets.clients) - for (client of Array.from(clientList)) { - if (!seen[client.id]) { - seen[client.id] = true; - if (client.publicId === update.meta.source) { - logger.log({doc_id, version: update.v, source: (update.meta != null ? update.meta.source : undefined)}, "distributing update to sender"); - client.emit("otUpdateApplied", {v: update.v, doc: update.doc}); - } else if (!update.dup) { // Duplicate ops should just be sent back to sending client for acknowledgement - logger.log({doc_id, version: update.v, source: (update.meta != null ? update.meta.source : undefined), client_id: client.id}, "distributing update to collaborator"); - client.emit("otUpdateApplied", update); - } - } - } - if (Object.keys(seen).length < clientList.length) { - metrics.inc("socket-io.duplicate-clients", 0.1); - return logger.log({doc_id, socketIoClients: (((() => { - const result1 = []; - for (client of Array.from(clientList)) { result1.push(client.id); - } - return result1; - })()))}, "discarded duplicate clients"); - } - }, - - _processErrorFromDocumentUpdater(io, doc_id, error, message) { - return (() => { - const result = []; - for (const client of Array.from(io.sockets.clients(doc_id))) { - logger.warn({err: error, doc_id, client_id: client.id}, "error from document updater, disconnecting client"); - client.emit("otUpdateError", error, message); - result.push(client.disconnect()); - } - return result; - })(); - } -}); + _processMessageFromDocumentUpdater(io, channel, message) { + return SafeJsonParse.parse(message, function (error, message) { + if (error != null) { + logger.error({ err: error, channel }, 'error parsing JSON') + return + } + if (message.op != null) { + if (message._id != null && settings.checkEventOrder) { + const status = EventLogger.checkEventOrder( + 'applied-ops', + message._id, + message + ) + if (status === 'duplicate') { + return // skip duplicate events + } + } + return DocumentUpdaterController._applyUpdateFromDocumentUpdater( + io, + message.doc_id, + message.op + ) + } else if (message.error != null) { + return DocumentUpdaterController._processErrorFromDocumentUpdater( + io, + message.doc_id, + message.error, + message + ) + } else if (message.health_check != null) { + logger.debug( + { message }, + 'got health check message in applied ops channel' + ) + return HealthCheckManager.check(channel, message.key) + } + }) + }, + _applyUpdateFromDocumentUpdater(io, doc_id, update) { + let client + const clientList = io.sockets.clients(doc_id) + // avoid unnecessary work if no clients are connected + if (clientList.length === 0) { + return + } + // send updates to clients + logger.log( + { + doc_id, + version: update.v, + source: update.meta != null ? update.meta.source : undefined, + socketIoClients: (() => { + const result = [] + for (client of Array.from(clientList)) { + result.push(client.id) + } + return result + })() + }, + 'distributing updates to clients' + ) + const seen = {} + // send messages only to unique clients (due to duplicate entries in io.sockets.clients) + for (client of Array.from(clientList)) { + if (!seen[client.id]) { + seen[client.id] = true + if (client.publicId === update.meta.source) { + logger.log( + { + doc_id, + version: update.v, + source: update.meta != null ? update.meta.source : undefined + }, + 'distributing update to sender' + ) + client.emit('otUpdateApplied', { v: update.v, doc: update.doc }) + } else if (!update.dup) { + // Duplicate ops should just be sent back to sending client for acknowledgement + logger.log( + { + doc_id, + version: update.v, + source: update.meta != null ? update.meta.source : undefined, + client_id: client.id + }, + 'distributing update to collaborator' + ) + client.emit('otUpdateApplied', update) + } + } + } + if (Object.keys(seen).length < clientList.length) { + metrics.inc('socket-io.duplicate-clients', 0.1) + return logger.log( + { + doc_id, + socketIoClients: (() => { + const result1 = [] + for (client of Array.from(clientList)) { + result1.push(client.id) + } + return result1 + })() + }, + 'discarded duplicate clients' + ) + } + }, + _processErrorFromDocumentUpdater(io, doc_id, error, message) { + return (() => { + const result = [] + for (const client of Array.from(io.sockets.clients(doc_id))) { + logger.warn( + { err: error, doc_id, client_id: client.id }, + 'error from document updater, disconnecting client' + ) + client.emit('otUpdateError', error, message) + result.push(client.disconnect()) + } + return result + })() + } +} diff --git a/services/real-time/app/js/DocumentUpdaterManager.js b/services/real-time/app/js/DocumentUpdaterManager.js index dc5865db62..bafc81ed14 100644 --- a/services/real-time/app/js/DocumentUpdaterManager.js +++ b/services/real-time/app/js/DocumentUpdaterManager.js @@ -11,104 +11,159 @@ * DS207: Consider shorter variations of null checks * Full docs: https://github.com/decaffeinate/decaffeinate/blob/master/docs/suggestions.md */ -let DocumentUpdaterManager; -const request = require("request"); -const _ = require("underscore"); -const logger = require("logger-sharelatex"); -const settings = require("settings-sharelatex"); -const metrics = require("metrics-sharelatex"); +let DocumentUpdaterManager +const request = require('request') +const _ = require('underscore') +const logger = require('logger-sharelatex') +const settings = require('settings-sharelatex') +const metrics = require('metrics-sharelatex') -const rclient = require("redis-sharelatex").createClient(settings.redis.documentupdater); -const Keys = settings.redis.documentupdater.key_schema; +const rclient = require('redis-sharelatex').createClient( + settings.redis.documentupdater +) +const Keys = settings.redis.documentupdater.key_schema -module.exports = (DocumentUpdaterManager = { - getDocument(project_id, doc_id, fromVersion, callback) { - if (callback == null) { callback = function(error, exists, doclines, version) {}; } - const timer = new metrics.Timer("get-document"); - const url = `${settings.apis.documentupdater.url}/project/${project_id}/doc/${doc_id}?fromVersion=${fromVersion}`; - logger.log({project_id, doc_id, fromVersion}, "getting doc from document updater"); - return request.get(url, function(err, res, body) { - timer.done(); - if (err != null) { - logger.error({err, url, project_id, doc_id}, "error getting doc from doc updater"); - return callback(err); - } - if (res.statusCode >= 200 && res.statusCode < 300) { - logger.log({project_id, doc_id}, "got doc from document document updater"); - try { - body = JSON.parse(body); - } catch (error) { - return callback(error); - } - return callback(null, body != null ? body.lines : undefined, body != null ? body.version : undefined, body != null ? body.ranges : undefined, body != null ? body.ops : undefined); - } else if ([404, 422].includes(res.statusCode)) { - err = new Error("doc updater could not load requested ops"); - err.statusCode = res.statusCode; - logger.warn({err, project_id, doc_id, url, fromVersion}, "doc updater could not load requested ops"); - return callback(err); - } else { - err = new Error(`doc updater returned a non-success status code: ${res.statusCode}`); - err.statusCode = res.statusCode; - logger.error({err, project_id, doc_id, url}, `doc updater returned a non-success status code: ${res.statusCode}`); - return callback(err); - } - }); - }, +module.exports = DocumentUpdaterManager = { + getDocument(project_id, doc_id, fromVersion, callback) { + if (callback == null) { + callback = function (error, exists, doclines, version) {} + } + const timer = new metrics.Timer('get-document') + const url = `${settings.apis.documentupdater.url}/project/${project_id}/doc/${doc_id}?fromVersion=${fromVersion}` + logger.log( + { project_id, doc_id, fromVersion }, + 'getting doc from document updater' + ) + return request.get(url, function (err, res, body) { + timer.done() + if (err != null) { + logger.error( + { err, url, project_id, doc_id }, + 'error getting doc from doc updater' + ) + return callback(err) + } + if (res.statusCode >= 200 && res.statusCode < 300) { + logger.log( + { project_id, doc_id }, + 'got doc from document document updater' + ) + try { + body = JSON.parse(body) + } catch (error) { + return callback(error) + } + return callback( + null, + body != null ? body.lines : undefined, + body != null ? body.version : undefined, + body != null ? body.ranges : undefined, + body != null ? body.ops : undefined + ) + } else if ([404, 422].includes(res.statusCode)) { + err = new Error('doc updater could not load requested ops') + err.statusCode = res.statusCode + logger.warn( + { err, project_id, doc_id, url, fromVersion }, + 'doc updater could not load requested ops' + ) + return callback(err) + } else { + err = new Error( + `doc updater returned a non-success status code: ${res.statusCode}` + ) + err.statusCode = res.statusCode + logger.error( + { err, project_id, doc_id, url }, + `doc updater returned a non-success status code: ${res.statusCode}` + ) + return callback(err) + } + }) + }, - flushProjectToMongoAndDelete(project_id, callback) { - // this method is called when the last connected user leaves the project - if (callback == null) { callback = function(){}; } - logger.log({project_id}, "deleting project from document updater"); - const timer = new metrics.Timer("delete.mongo.project"); - // flush the project in the background when all users have left - const url = `${settings.apis.documentupdater.url}/project/${project_id}?background=true` + - (settings.shutDownInProgress ? "&shutdown=true" : ""); - return request.del(url, function(err, res, body){ - timer.done(); - if (err != null) { - logger.error({err, project_id}, "error deleting project from document updater"); - return callback(err); - } else if (res.statusCode >= 200 && res.statusCode < 300) { - logger.log({project_id}, "deleted project from document updater"); - return callback(null); - } else { - err = new Error(`document updater returned a failure status code: ${res.statusCode}`); - err.statusCode = res.statusCode; - logger.error({err, project_id}, `document updater returned failure status code: ${res.statusCode}`); - return callback(err); - } - }); - }, + flushProjectToMongoAndDelete(project_id, callback) { + // this method is called when the last connected user leaves the project + if (callback == null) { + callback = function () {} + } + logger.log({ project_id }, 'deleting project from document updater') + const timer = new metrics.Timer('delete.mongo.project') + // flush the project in the background when all users have left + const url = + `${settings.apis.documentupdater.url}/project/${project_id}?background=true` + + (settings.shutDownInProgress ? '&shutdown=true' : '') + return request.del(url, function (err, res, body) { + timer.done() + if (err != null) { + logger.error( + { err, project_id }, + 'error deleting project from document updater' + ) + return callback(err) + } else if (res.statusCode >= 200 && res.statusCode < 300) { + logger.log({ project_id }, 'deleted project from document updater') + return callback(null) + } else { + err = new Error( + `document updater returned a failure status code: ${res.statusCode}` + ) + err.statusCode = res.statusCode + logger.error( + { err, project_id }, + `document updater returned failure status code: ${res.statusCode}` + ) + return callback(err) + } + }) + }, - queueChange(project_id, doc_id, change, callback){ - let error; - if (callback == null) { callback = function(){}; } - const allowedKeys = [ 'doc', 'op', 'v', 'dupIfSource', 'meta', 'lastV', 'hash']; - change = _.pick(change, allowedKeys); - const jsonChange = JSON.stringify(change); - if (jsonChange.indexOf("\u0000") !== -1) { - // memory corruption check - error = new Error("null bytes found in op"); - logger.error({err: error, project_id, doc_id, jsonChange}, error.message); - return callback(error); - } + queueChange(project_id, doc_id, change, callback) { + let error + if (callback == null) { + callback = function () {} + } + const allowedKeys = [ + 'doc', + 'op', + 'v', + 'dupIfSource', + 'meta', + 'lastV', + 'hash' + ] + change = _.pick(change, allowedKeys) + const jsonChange = JSON.stringify(change) + if (jsonChange.indexOf('\u0000') !== -1) { + // memory corruption check + error = new Error('null bytes found in op') + logger.error( + { err: error, project_id, doc_id, jsonChange }, + error.message + ) + return callback(error) + } - const updateSize = jsonChange.length; - if (updateSize > settings.maxUpdateSize) { - error = new Error("update is too large"); - error.updateSize = updateSize; - return callback(error); - } + const updateSize = jsonChange.length + if (updateSize > settings.maxUpdateSize) { + error = new Error('update is too large') + error.updateSize = updateSize + return callback(error) + } - // record metric for each update added to queue - metrics.summary('redis.pendingUpdates', updateSize, {status: 'push'}); + // record metric for each update added to queue + metrics.summary('redis.pendingUpdates', updateSize, { status: 'push' }) - const doc_key = `${project_id}:${doc_id}`; - // Push onto pendingUpdates for doc_id first, because once the doc updater - // gets an entry on pending-updates-list, it starts processing. - return rclient.rpush(Keys.pendingUpdates({doc_id}), jsonChange, function(error) { - if (error != null) { return callback(error); } - return rclient.rpush("pending-updates-list", doc_key, callback); - }); - } -}); + const doc_key = `${project_id}:${doc_id}` + // Push onto pendingUpdates for doc_id first, because once the doc updater + // gets an entry on pending-updates-list, it starts processing. + return rclient.rpush(Keys.pendingUpdates({ doc_id }), jsonChange, function ( + error + ) { + if (error != null) { + return callback(error) + } + return rclient.rpush('pending-updates-list', doc_key, callback) + }) + } +} diff --git a/services/real-time/app/js/DrainManager.js b/services/real-time/app/js/DrainManager.js index 466c80fd0c..b8c08356bb 100644 --- a/services/real-time/app/js/DrainManager.js +++ b/services/real-time/app/js/DrainManager.js @@ -9,54 +9,55 @@ * DS102: Remove unnecessary code created because of implicit returns * Full docs: https://github.com/decaffeinate/decaffeinate/blob/master/docs/suggestions.md */ -let DrainManager; -const logger = require("logger-sharelatex"); +let DrainManager +const logger = require('logger-sharelatex') -module.exports = (DrainManager = { +module.exports = DrainManager = { + startDrainTimeWindow(io, minsToDrain) { + const drainPerMin = io.sockets.clients().length / minsToDrain + return DrainManager.startDrain(io, Math.max(drainPerMin / 60, 4)) + }, // enforce minimum drain rate - startDrainTimeWindow(io, minsToDrain){ - const drainPerMin = io.sockets.clients().length / minsToDrain; - return DrainManager.startDrain(io, Math.max(drainPerMin / 60, 4)); - }, // enforce minimum drain rate + startDrain(io, rate) { + // Clear out any old interval + let pollingInterval + clearInterval(this.interval) + logger.log({ rate }, 'starting drain') + if (rate === 0) { + return + } else if (rate < 1) { + // allow lower drain rates + // e.g. rate=0.1 will drain one client every 10 seconds + pollingInterval = 1000 / rate + rate = 1 + } else { + pollingInterval = 1000 + } + return (this.interval = setInterval(() => { + return this.reconnectNClients(io, rate) + }, pollingInterval)) + }, - startDrain(io, rate) { - // Clear out any old interval - let pollingInterval; - clearInterval(this.interval); - logger.log({rate}, "starting drain"); - if (rate === 0) { - return; - } else if (rate < 1) { - // allow lower drain rates - // e.g. rate=0.1 will drain one client every 10 seconds - pollingInterval = 1000 / rate; - rate = 1; - } else { - pollingInterval = 1000; - } - return this.interval = setInterval(() => { - return this.reconnectNClients(io, rate); - } - , pollingInterval); - }, - - RECONNECTED_CLIENTS: {}, - reconnectNClients(io, N) { - let drainedCount = 0; - for (const client of Array.from(io.sockets.clients())) { - if (!this.RECONNECTED_CLIENTS[client.id]) { - this.RECONNECTED_CLIENTS[client.id] = true; - logger.log({client_id: client.id}, "Asking client to reconnect gracefully"); - client.emit("reconnectGracefully"); - drainedCount++; - } - const haveDrainedNClients = (drainedCount === N); - if (haveDrainedNClients) { - break; - } - } - if (drainedCount < N) { - return logger.log("All clients have been told to reconnectGracefully"); - } - } -}); + RECONNECTED_CLIENTS: {}, + reconnectNClients(io, N) { + let drainedCount = 0 + for (const client of Array.from(io.sockets.clients())) { + if (!this.RECONNECTED_CLIENTS[client.id]) { + this.RECONNECTED_CLIENTS[client.id] = true + logger.log( + { client_id: client.id }, + 'Asking client to reconnect gracefully' + ) + client.emit('reconnectGracefully') + drainedCount++ + } + const haveDrainedNClients = drainedCount === N + if (haveDrainedNClients) { + break + } + } + if (drainedCount < N) { + return logger.log('All clients have been told to reconnectGracefully') + } + } +} diff --git a/services/real-time/app/js/Errors.js b/services/real-time/app/js/Errors.js index 04437742fb..8bfe3763b0 100644 --- a/services/real-time/app/js/Errors.js +++ b/services/real-time/app/js/Errors.js @@ -4,15 +4,14 @@ */ // TODO: This file was created by bulk-decaffeinate. // Fix any style issues and re-enable lint. -let Errors; -var CodedError = function(message, code) { - const error = new Error(message); - error.name = "CodedError"; - error.code = code; - error.__proto__ = CodedError.prototype; - return error; -}; -CodedError.prototype.__proto__ = Error.prototype; +let Errors +var CodedError = function (message, code) { + const error = new Error(message) + error.name = 'CodedError' + error.code = code + error.__proto__ = CodedError.prototype + return error +} +CodedError.prototype.__proto__ = Error.prototype -module.exports = (Errors = - {CodedError}); +module.exports = Errors = { CodedError } diff --git a/services/real-time/app/js/EventLogger.js b/services/real-time/app/js/EventLogger.js index 8a700326b5..1133ebdaf8 100644 --- a/services/real-time/app/js/EventLogger.js +++ b/services/real-time/app/js/EventLogger.js @@ -10,84 +10,91 @@ * DS207: Consider shorter variations of null checks * Full docs: https://github.com/decaffeinate/decaffeinate/blob/master/docs/suggestions.md */ -let EventLogger; -const logger = require('logger-sharelatex'); -const metrics = require('metrics-sharelatex'); -const settings = require('settings-sharelatex'); +let EventLogger +const logger = require('logger-sharelatex') +const metrics = require('metrics-sharelatex') +const settings = require('settings-sharelatex') // keep track of message counters to detect duplicate and out of order events // messsage ids have the format "UNIQUEHOSTKEY-COUNTER" -const EVENT_LOG_COUNTER = {}; -const EVENT_LOG_TIMESTAMP = {}; -let EVENT_LAST_CLEAN_TIMESTAMP = 0; +const EVENT_LOG_COUNTER = {} +const EVENT_LOG_TIMESTAMP = {} +let EVENT_LAST_CLEAN_TIMESTAMP = 0 // counter for debug logs -let COUNTER = 0; +let COUNTER = 0 -module.exports = (EventLogger = { +module.exports = EventLogger = { + MAX_STALE_TIME_IN_MS: 3600 * 1000, - MAX_STALE_TIME_IN_MS: 3600 * 1000, + debugEvent(channel, message) { + if (settings.debugEvents > 0) { + logger.log({ channel, message, counter: COUNTER++ }, 'logging event') + return settings.debugEvents-- + } + }, - debugEvent(channel, message) { - if (settings.debugEvents > 0) { - logger.log({channel, message, counter: COUNTER++}, "logging event"); - return settings.debugEvents--; - } - }, + checkEventOrder(channel, message_id, message) { + let result + if (typeof message_id !== 'string') { + return + } + if (!(result = message_id.match(/^(.*)-(\d+)$/))) { + return + } + const key = result[1] + const count = parseInt(result[2], 0) + if (!(count >= 0)) { + // ignore checks if counter is not present + return + } + // store the last count in a hash for each host + const previous = EventLogger._storeEventCount(key, count) + if (previous == null || count === previous + 1) { + metrics.inc(`event.${channel}.valid`, 0.001) // downsample high rate docupdater events + return // order is ok + } + if (count === previous) { + metrics.inc(`event.${channel}.duplicate`) + logger.warn({ channel, message_id }, 'duplicate event') + return 'duplicate' + } else { + metrics.inc(`event.${channel}.out-of-order`) + logger.warn( + { channel, message_id, key, previous, count }, + 'out of order event' + ) + return 'out-of-order' + } + }, - checkEventOrder(channel, message_id, message) { - let result; - if (typeof(message_id) !== 'string') { return; } - if (!(result = message_id.match(/^(.*)-(\d+)$/))) { return; } - const key = result[1]; - const count = parseInt(result[2], 0); - if (!(count >= 0)) {// ignore checks if counter is not present - return; - } - // store the last count in a hash for each host - const previous = EventLogger._storeEventCount(key, count); - if ((previous == null) || (count === (previous + 1))) { - metrics.inc(`event.${channel}.valid`, 0.001); // downsample high rate docupdater events - return; // order is ok - } - if (count === previous) { - metrics.inc(`event.${channel}.duplicate`); - logger.warn({channel, message_id}, "duplicate event"); - return "duplicate"; - } else { - metrics.inc(`event.${channel}.out-of-order`); - logger.warn({channel, message_id, key, previous, count}, "out of order event"); - return "out-of-order"; - } - }, + _storeEventCount(key, count) { + const previous = EVENT_LOG_COUNTER[key] + const now = Date.now() + EVENT_LOG_COUNTER[key] = count + EVENT_LOG_TIMESTAMP[key] = now + // periodically remove old counts + if (now - EVENT_LAST_CLEAN_TIMESTAMP > EventLogger.MAX_STALE_TIME_IN_MS) { + EventLogger._cleanEventStream(now) + EVENT_LAST_CLEAN_TIMESTAMP = now + } + return previous + }, - _storeEventCount(key, count) { - const previous = EVENT_LOG_COUNTER[key]; - const now = Date.now(); - EVENT_LOG_COUNTER[key] = count; - EVENT_LOG_TIMESTAMP[key] = now; - // periodically remove old counts - if ((now - EVENT_LAST_CLEAN_TIMESTAMP) > EventLogger.MAX_STALE_TIME_IN_MS) { - EventLogger._cleanEventStream(now); - EVENT_LAST_CLEAN_TIMESTAMP = now; - } - return previous; - }, - - _cleanEventStream(now) { - return (() => { - const result = []; - for (const key in EVENT_LOG_TIMESTAMP) { - const timestamp = EVENT_LOG_TIMESTAMP[key]; - if ((now - timestamp) > EventLogger.MAX_STALE_TIME_IN_MS) { - delete EVENT_LOG_COUNTER[key]; - result.push(delete EVENT_LOG_TIMESTAMP[key]); - } else { - result.push(undefined); - } - } - return result; - })(); - } -}); \ No newline at end of file + _cleanEventStream(now) { + return (() => { + const result = [] + for (const key in EVENT_LOG_TIMESTAMP) { + const timestamp = EVENT_LOG_TIMESTAMP[key] + if (now - timestamp > EventLogger.MAX_STALE_TIME_IN_MS) { + delete EVENT_LOG_COUNTER[key] + result.push(delete EVENT_LOG_TIMESTAMP[key]) + } else { + result.push(undefined) + } + } + return result + })() + } +} diff --git a/services/real-time/app/js/HealthCheckManager.js b/services/real-time/app/js/HealthCheckManager.js index f8a9aa672e..4704aa5e88 100644 --- a/services/real-time/app/js/HealthCheckManager.js +++ b/services/real-time/app/js/HealthCheckManager.js @@ -10,76 +10,84 @@ * DS207: Consider shorter variations of null checks * Full docs: https://github.com/decaffeinate/decaffeinate/blob/master/docs/suggestions.md */ -let HealthCheckManager; -const metrics = require("metrics-sharelatex"); -const logger = require("logger-sharelatex"); +let HealthCheckManager +const metrics = require('metrics-sharelatex') +const logger = require('logger-sharelatex') -const os = require("os"); -const HOST = os.hostname(); -const PID = process.pid; -let COUNT = 0; +const os = require('os') +const HOST = os.hostname() +const PID = process.pid +let COUNT = 0 -const CHANNEL_MANAGER = {}; // hash of event checkers by channel name -const CHANNEL_ERROR = {}; // error status by channel name +const CHANNEL_MANAGER = {} // hash of event checkers by channel name +const CHANNEL_ERROR = {} // error status by channel name -module.exports = (HealthCheckManager = class HealthCheckManager { - // create an instance of this class which checks that an event with a unique - // id is received only once within a timeout - constructor(channel, timeout) { - // unique event string - this.channel = channel; - if (timeout == null) { timeout = 1000; } - this.id = `host=${HOST}:pid=${PID}:count=${COUNT++}`; - // count of number of times the event is received - this.count = 0; - // after a timeout check the status of the count - this.handler = setTimeout(() => { - return this.setStatus(); - } - , timeout); - // use a timer to record the latency of the channel - this.timer = new metrics.Timer(`event.${this.channel}.latency`); - // keep a record of these objects to dispatch on - CHANNEL_MANAGER[this.channel] = this; +module.exports = HealthCheckManager = class HealthCheckManager { + // create an instance of this class which checks that an event with a unique + // id is received only once within a timeout + constructor(channel, timeout) { + // unique event string + this.channel = channel + if (timeout == null) { + timeout = 1000 } + this.id = `host=${HOST}:pid=${PID}:count=${COUNT++}` + // count of number of times the event is received + this.count = 0 + // after a timeout check the status of the count + this.handler = setTimeout(() => { + return this.setStatus() + }, timeout) + // use a timer to record the latency of the channel + this.timer = new metrics.Timer(`event.${this.channel}.latency`) + // keep a record of these objects to dispatch on + CHANNEL_MANAGER[this.channel] = this + } - processEvent(id) { - // if this is our event record it - if (id === this.id) { - this.count++; - if (this.timer != null) { - this.timer.done(); - } - return this.timer = null; // only time the latency of the first event - } + processEvent(id) { + // if this is our event record it + if (id === this.id) { + this.count++ + if (this.timer != null) { + this.timer.done() + } + return (this.timer = null) // only time the latency of the first event } + } - setStatus() { - // if we saw the event anything other than a single time that is an error - if (this.count !== 1) { - logger.err({channel:this.channel, count:this.count, id:this.id}, "redis channel health check error"); - } - const error = (this.count !== 1); - return CHANNEL_ERROR[this.channel] = error; + setStatus() { + // if we saw the event anything other than a single time that is an error + if (this.count !== 1) { + logger.err( + { channel: this.channel, count: this.count, id: this.id }, + 'redis channel health check error' + ) } + const error = this.count !== 1 + return (CHANNEL_ERROR[this.channel] = error) + } - // class methods - static check(channel, id) { - // dispatch event to manager for channel - return (CHANNEL_MANAGER[channel] != null ? CHANNEL_MANAGER[channel].processEvent(id) : undefined); - } + // class methods + static check(channel, id) { + // dispatch event to manager for channel + return CHANNEL_MANAGER[channel] != null + ? CHANNEL_MANAGER[channel].processEvent(id) + : undefined + } - static status() { - // return status of all channels for logging - return CHANNEL_ERROR; - } + static status() { + // return status of all channels for logging + return CHANNEL_ERROR + } - static isFailing() { - // check if any channel status is bad - for (const channel in CHANNEL_ERROR) { - const error = CHANNEL_ERROR[channel]; - if (error === true) { return true; } - } - return false; + static isFailing() { + // check if any channel status is bad + for (const channel in CHANNEL_ERROR) { + const error = CHANNEL_ERROR[channel] + if (error === true) { + return true + } } -}); + return false + } +} diff --git a/services/real-time/app/js/HttpApiController.js b/services/real-time/app/js/HttpApiController.js index 88bbc1a5e3..a512961797 100644 --- a/services/real-time/app/js/HttpApiController.js +++ b/services/real-time/app/js/HttpApiController.js @@ -10,47 +10,53 @@ * DS102: Remove unnecessary code created because of implicit returns * Full docs: https://github.com/decaffeinate/decaffeinate/blob/master/docs/suggestions.md */ -let HttpApiController; -const WebsocketLoadBalancer = require("./WebsocketLoadBalancer"); -const DrainManager = require("./DrainManager"); -const logger = require("logger-sharelatex"); +let HttpApiController +const WebsocketLoadBalancer = require('./WebsocketLoadBalancer') +const DrainManager = require('./DrainManager') +const logger = require('logger-sharelatex') -module.exports = (HttpApiController = { - sendMessage(req, res, next) { - logger.log({message: req.params.message}, "sending message"); - if (Array.isArray(req.body)) { - for (const payload of Array.from(req.body)) { - WebsocketLoadBalancer.emitToRoom(req.params.project_id, req.params.message, payload); - } - } else { - WebsocketLoadBalancer.emitToRoom(req.params.project_id, req.params.message, req.body); - } - return res.send(204); - }, // No content - - startDrain(req, res, next) { - const io = req.app.get("io"); - let rate = req.query.rate || "4"; - rate = parseFloat(rate) || 0; - logger.log({rate}, "setting client drain rate"); - DrainManager.startDrain(io, rate); - return res.send(204); - }, +module.exports = HttpApiController = { + sendMessage(req, res, next) { + logger.log({ message: req.params.message }, 'sending message') + if (Array.isArray(req.body)) { + for (const payload of Array.from(req.body)) { + WebsocketLoadBalancer.emitToRoom( + req.params.project_id, + req.params.message, + payload + ) + } + } else { + WebsocketLoadBalancer.emitToRoom( + req.params.project_id, + req.params.message, + req.body + ) + } + return res.send(204) + }, // No content - disconnectClient(req, res, next) { - const io = req.app.get("io"); - const { - client_id - } = req.params; - const client = io.sockets.sockets[client_id]; + startDrain(req, res, next) { + const io = req.app.get('io') + let rate = req.query.rate || '4' + rate = parseFloat(rate) || 0 + logger.log({ rate }, 'setting client drain rate') + DrainManager.startDrain(io, rate) + return res.send(204) + }, - if (!client) { - logger.info({client_id}, "api: client already disconnected"); - res.sendStatus(404); - return; - } - logger.warn({client_id}, "api: requesting client disconnect"); - client.on("disconnect", () => res.sendStatus(204)); - return client.disconnect(); - } -}); + disconnectClient(req, res, next) { + const io = req.app.get('io') + const { client_id } = req.params + const client = io.sockets.sockets[client_id] + + if (!client) { + logger.info({ client_id }, 'api: client already disconnected') + res.sendStatus(404) + return + } + logger.warn({ client_id }, 'api: requesting client disconnect') + client.on('disconnect', () => res.sendStatus(204)) + return client.disconnect() + } +} diff --git a/services/real-time/app/js/HttpController.js b/services/real-time/app/js/HttpController.js index 4d33af44b3..deabf5876d 100644 --- a/services/real-time/app/js/HttpController.js +++ b/services/real-time/app/js/HttpController.js @@ -10,50 +10,78 @@ * DS207: Consider shorter variations of null checks * Full docs: https://github.com/decaffeinate/decaffeinate/blob/master/docs/suggestions.md */ -let HttpController; -const async = require("async"); +let HttpController +const async = require('async') -module.exports = (HttpController = { - // The code in this controller is hard to unit test because of a lot of - // dependencies on internal socket.io methods. It is not critical to the running - // of ShareLaTeX, and is only used for getting stats about connected clients, - // and for checking internal state in acceptance tests. The acceptances tests - // should provide appropriate coverage. - _getConnectedClientView(ioClient, callback) { - if (callback == null) { callback = function(error, client) {}; } - const client_id = ioClient.id; - const {project_id, user_id, first_name, last_name, email, connected_time} = ioClient.ol_context; - const client = {client_id, project_id, user_id, first_name, last_name, email, connected_time}; - client.rooms = []; - for (const name in ioClient.manager.roomClients[client_id]) { - const joined = ioClient.manager.roomClients[client_id][name]; - if (joined && (name !== "")) { - client.rooms.push(name.replace(/^\//, "")); // Remove leading / - } - } - return callback(null, client); - }, +module.exports = HttpController = { + // The code in this controller is hard to unit test because of a lot of + // dependencies on internal socket.io methods. It is not critical to the running + // of ShareLaTeX, and is only used for getting stats about connected clients, + // and for checking internal state in acceptance tests. The acceptances tests + // should provide appropriate coverage. + _getConnectedClientView(ioClient, callback) { + if (callback == null) { + callback = function (error, client) {} + } + const client_id = ioClient.id + const { + project_id, + user_id, + first_name, + last_name, + email, + connected_time + } = ioClient.ol_context + const client = { + client_id, + project_id, + user_id, + first_name, + last_name, + email, + connected_time + } + client.rooms = [] + for (const name in ioClient.manager.roomClients[client_id]) { + const joined = ioClient.manager.roomClients[client_id][name] + if (joined && name !== '') { + client.rooms.push(name.replace(/^\//, '')) // Remove leading / + } + } + return callback(null, client) + }, - getConnectedClients(req, res, next) { - const io = req.app.get("io"); - const ioClients = io.sockets.clients(); - return async.map(ioClients, HttpController._getConnectedClientView, function(error, clients) { - if (error != null) { return next(error); } - return res.json(clients); - }); - }, - - getConnectedClient(req, res, next) { - const {client_id} = req.params; - const io = req.app.get("io"); - const ioClient = io.sockets.sockets[client_id]; - if (!ioClient) { - res.sendStatus(404); - return; - } - return HttpController._getConnectedClientView(ioClient, function(error, client) { - if (error != null) { return next(error); } - return res.json(client); - }); - } -}); + getConnectedClients(req, res, next) { + const io = req.app.get('io') + const ioClients = io.sockets.clients() + return async.map( + ioClients, + HttpController._getConnectedClientView, + function (error, clients) { + if (error != null) { + return next(error) + } + return res.json(clients) + } + ) + }, + + getConnectedClient(req, res, next) { + const { client_id } = req.params + const io = req.app.get('io') + const ioClient = io.sockets.sockets[client_id] + if (!ioClient) { + res.sendStatus(404) + return + } + return HttpController._getConnectedClientView(ioClient, function ( + error, + client + ) { + if (error != null) { + return next(error) + } + return res.json(client) + }) + } +} diff --git a/services/real-time/app/js/RedisClientManager.js b/services/real-time/app/js/RedisClientManager.js index 3da2136b46..b43262aeda 100644 --- a/services/real-time/app/js/RedisClientManager.js +++ b/services/real-time/app/js/RedisClientManager.js @@ -10,31 +10,31 @@ * DS207: Consider shorter variations of null checks * Full docs: https://github.com/decaffeinate/decaffeinate/blob/master/docs/suggestions.md */ -let RedisClientManager; -const redis = require("redis-sharelatex"); -const logger = require('logger-sharelatex'); +let RedisClientManager +const redis = require('redis-sharelatex') +const logger = require('logger-sharelatex') -module.exports = (RedisClientManager = { - createClientList(...configs) { - // create a dynamic list of redis clients, excluding any configurations which are not defined - const clientList = (() => { - const result = []; - for (const x of Array.from(configs)) { - if (x != null) { - const redisType = (x.cluster != null) ? - "cluster" - : (x.sentinels != null) ? - "sentinel" - : (x.host != null) ? - "single" - : - "unknown"; - logger.log({redis: redisType}, "creating redis client"); - result.push(redis.createClient(x)); - } - } - return result; - })(); - return clientList; - } -}); \ No newline at end of file +module.exports = RedisClientManager = { + createClientList(...configs) { + // create a dynamic list of redis clients, excluding any configurations which are not defined + const clientList = (() => { + const result = [] + for (const x of Array.from(configs)) { + if (x != null) { + const redisType = + x.cluster != null + ? 'cluster' + : x.sentinels != null + ? 'sentinel' + : x.host != null + ? 'single' + : 'unknown' + logger.log({ redis: redisType }, 'creating redis client') + result.push(redis.createClient(x)) + } + } + return result + })() + return clientList + } +} diff --git a/services/real-time/app/js/RoomManager.js b/services/real-time/app/js/RoomManager.js index c75cc68626..8dd34e9340 100644 --- a/services/real-time/app/js/RoomManager.js +++ b/services/real-time/app/js/RoomManager.js @@ -13,13 +13,13 @@ * DS207: Consider shorter variations of null checks * Full docs: https://github.com/decaffeinate/decaffeinate/blob/master/docs/suggestions.md */ -let RoomManager; -const logger = require('logger-sharelatex'); -const metrics = require("metrics-sharelatex"); -const {EventEmitter} = require('events'); +let RoomManager +const logger = require('logger-sharelatex') +const metrics = require('metrics-sharelatex') +const { EventEmitter } = require('events') -const IdMap = new Map(); // keep track of whether ids are from projects or docs -const RoomEvents = new EventEmitter(); // emits {project,doc}-active and {project,doc}-empty events +const IdMap = new Map() // keep track of whether ids are from projects or docs +const RoomEvents = new EventEmitter() // emits {project,doc}-active and {project,doc}-empty events // Manage socket.io rooms for individual projects and docs // @@ -31,130 +31,159 @@ const RoomEvents = new EventEmitter(); // emits {project,doc}-active and {projec // // The pubsub side is handled by ChannelManager -module.exports = (RoomManager = { - - joinProject(client, project_id, callback) { - if (callback == null) { callback = function() {}; } - return this.joinEntity(client, "project", project_id, callback); - }, - - joinDoc(client, doc_id, callback) { - if (callback == null) { callback = function() {}; } - return this.joinEntity(client, "doc", doc_id, callback); - }, - - leaveDoc(client, doc_id) { - return this.leaveEntity(client, "doc", doc_id); - }, - - leaveProjectAndDocs(client) { - // what rooms is this client in? we need to leave them all. socket.io - // will cause us to leave the rooms, so we only need to manage our - // channel subscriptions... but it will be safer if we leave them - // explicitly, and then socket.io will just regard this as a client that - // has not joined any rooms and do a final disconnection. - const roomsToLeave = this._roomsClientIsIn(client); - logger.log({client: client.id, roomsToLeave}, "client leaving project"); - return (() => { - const result = []; - for (const id of Array.from(roomsToLeave)) { - const entity = IdMap.get(id); - result.push(this.leaveEntity(client, entity, id)); - } - return result; - })(); - }, - - emitOnCompletion(promiseList, eventName) { - return Promise.all(promiseList) - .then(() => RoomEvents.emit(eventName)) - .catch(err => RoomEvents.emit(eventName, err)); - }, - - eventSource() { - return RoomEvents; - }, - - joinEntity(client, entity, id, callback) { - const beforeCount = this._clientsInRoom(client, id); - // client joins room immediately but joinDoc request does not complete - // until room is subscribed - client.join(id); - // is this a new room? if so, subscribe - if (beforeCount === 0) { - logger.log({entity, id}, "room is now active"); - RoomEvents.once(`${entity}-subscribed-${id}`, function(err) { - // only allow the client to join when all the relevant channels have subscribed - logger.log({client: client.id, entity, id, beforeCount}, "client joined new room and subscribed to channel"); - return callback(err); - }); - RoomEvents.emit(`${entity}-active`, id); - IdMap.set(id, entity); - // keep track of the number of listeners - return metrics.gauge("room-listeners", RoomEvents.eventNames().length); - } else { - logger.log({client: client.id, entity, id, beforeCount}, "client joined existing room"); - client.join(id); - return callback(); - } - }, - - leaveEntity(client, entity, id) { - // Ignore any requests to leave when the client is not actually in the - // room. This can happen if the client sends spurious leaveDoc requests - // for old docs after a reconnection. - // This can now happen all the time, as we skip the join for clients that - // disconnect before joinProject/joinDoc completed. - if (!this._clientAlreadyInRoom(client, id)) { - logger.log({client: client.id, entity, id}, "ignoring request from client to leave room it is not in"); - return; - } - client.leave(id); - const afterCount = this._clientsInRoom(client, id); - logger.log({client: client.id, entity, id, afterCount}, "client left room"); - // is the room now empty? if so, unsubscribe - if ((entity == null)) { - logger.error({entity: id}, "unknown entity when leaving with id"); - return; - } - if (afterCount === 0) { - logger.log({entity, id}, "room is now empty"); - RoomEvents.emit(`${entity}-empty`, id); - IdMap.delete(id); - return metrics.gauge("room-listeners", RoomEvents.eventNames().length); - } - }, - - // internal functions below, these access socket.io rooms data directly and - // will need updating for socket.io v2 - - _clientsInRoom(client, room) { - const nsp = client.namespace.name; - const name = (nsp + '/') + room; - return (__guard__(client.manager != null ? client.manager.rooms : undefined, x => x[name]) || []).length; - }, - - _roomsClientIsIn(client) { - const roomList = (() => { - const result = []; - for (const fullRoomPath in (client.manager.roomClients != null ? client.manager.roomClients[client.id] : undefined)) { - // strip socket.io prefix from room to get original id - if (fullRoomPath !== '') { - const [prefix, room] = Array.from(fullRoomPath.split('/', 2)); - result.push(room); - } - } - return result; - })(); - return roomList; - }, - - _clientAlreadyInRoom(client, room) { - const nsp = client.namespace.name; - const name = (nsp + '/') + room; - return __guard__(client.manager.roomClients != null ? client.manager.roomClients[client.id] : undefined, x => x[name]); +module.exports = RoomManager = { + joinProject(client, project_id, callback) { + if (callback == null) { + callback = function () {} } -}); + return this.joinEntity(client, 'project', project_id, callback) + }, + + joinDoc(client, doc_id, callback) { + if (callback == null) { + callback = function () {} + } + return this.joinEntity(client, 'doc', doc_id, callback) + }, + + leaveDoc(client, doc_id) { + return this.leaveEntity(client, 'doc', doc_id) + }, + + leaveProjectAndDocs(client) { + // what rooms is this client in? we need to leave them all. socket.io + // will cause us to leave the rooms, so we only need to manage our + // channel subscriptions... but it will be safer if we leave them + // explicitly, and then socket.io will just regard this as a client that + // has not joined any rooms and do a final disconnection. + const roomsToLeave = this._roomsClientIsIn(client) + logger.log({ client: client.id, roomsToLeave }, 'client leaving project') + return (() => { + const result = [] + for (const id of Array.from(roomsToLeave)) { + const entity = IdMap.get(id) + result.push(this.leaveEntity(client, entity, id)) + } + return result + })() + }, + + emitOnCompletion(promiseList, eventName) { + return Promise.all(promiseList) + .then(() => RoomEvents.emit(eventName)) + .catch((err) => RoomEvents.emit(eventName, err)) + }, + + eventSource() { + return RoomEvents + }, + + joinEntity(client, entity, id, callback) { + const beforeCount = this._clientsInRoom(client, id) + // client joins room immediately but joinDoc request does not complete + // until room is subscribed + client.join(id) + // is this a new room? if so, subscribe + if (beforeCount === 0) { + logger.log({ entity, id }, 'room is now active') + RoomEvents.once(`${entity}-subscribed-${id}`, function (err) { + // only allow the client to join when all the relevant channels have subscribed + logger.log( + { client: client.id, entity, id, beforeCount }, + 'client joined new room and subscribed to channel' + ) + return callback(err) + }) + RoomEvents.emit(`${entity}-active`, id) + IdMap.set(id, entity) + // keep track of the number of listeners + return metrics.gauge('room-listeners', RoomEvents.eventNames().length) + } else { + logger.log( + { client: client.id, entity, id, beforeCount }, + 'client joined existing room' + ) + client.join(id) + return callback() + } + }, + + leaveEntity(client, entity, id) { + // Ignore any requests to leave when the client is not actually in the + // room. This can happen if the client sends spurious leaveDoc requests + // for old docs after a reconnection. + // This can now happen all the time, as we skip the join for clients that + // disconnect before joinProject/joinDoc completed. + if (!this._clientAlreadyInRoom(client, id)) { + logger.log( + { client: client.id, entity, id }, + 'ignoring request from client to leave room it is not in' + ) + return + } + client.leave(id) + const afterCount = this._clientsInRoom(client, id) + logger.log( + { client: client.id, entity, id, afterCount }, + 'client left room' + ) + // is the room now empty? if so, unsubscribe + if (entity == null) { + logger.error({ entity: id }, 'unknown entity when leaving with id') + return + } + if (afterCount === 0) { + logger.log({ entity, id }, 'room is now empty') + RoomEvents.emit(`${entity}-empty`, id) + IdMap.delete(id) + return metrics.gauge('room-listeners', RoomEvents.eventNames().length) + } + }, + + // internal functions below, these access socket.io rooms data directly and + // will need updating for socket.io v2 + + _clientsInRoom(client, room) { + const nsp = client.namespace.name + const name = nsp + '/' + room + return ( + __guard__( + client.manager != null ? client.manager.rooms : undefined, + (x) => x[name] + ) || [] + ).length + }, + + _roomsClientIsIn(client) { + const roomList = (() => { + const result = [] + for (const fullRoomPath in client.manager.roomClients != null + ? client.manager.roomClients[client.id] + : undefined) { + // strip socket.io prefix from room to get original id + if (fullRoomPath !== '') { + const [prefix, room] = Array.from(fullRoomPath.split('/', 2)) + result.push(room) + } + } + return result + })() + return roomList + }, + + _clientAlreadyInRoom(client, room) { + const nsp = client.namespace.name + const name = nsp + '/' + room + return __guard__( + client.manager.roomClients != null + ? client.manager.roomClients[client.id] + : undefined, + (x) => x[name] + ) + } +} function __guard__(value, transform) { - return (typeof value !== 'undefined' && value !== null) ? transform(value) : undefined; -} \ No newline at end of file + return typeof value !== 'undefined' && value !== null + ? transform(value) + : undefined +} diff --git a/services/real-time/app/js/Router.js b/services/real-time/app/js/Router.js index f475596036..0e19c46bc0 100644 --- a/services/real-time/app/js/Router.js +++ b/services/real-time/app/js/Router.js @@ -13,259 +13,390 @@ * DS207: Consider shorter variations of null checks * Full docs: https://github.com/decaffeinate/decaffeinate/blob/master/docs/suggestions.md */ -let Router; -const metrics = require("metrics-sharelatex"); -const logger = require("logger-sharelatex"); -const settings = require("settings-sharelatex"); -const WebsocketController = require("./WebsocketController"); -const HttpController = require("./HttpController"); -const HttpApiController = require("./HttpApiController"); -const bodyParser = require("body-parser"); -const base64id = require("base64id"); +let Router +const metrics = require('metrics-sharelatex') +const logger = require('logger-sharelatex') +const settings = require('settings-sharelatex') +const WebsocketController = require('./WebsocketController') +const HttpController = require('./HttpController') +const HttpApiController = require('./HttpApiController') +const bodyParser = require('body-parser') +const base64id = require('base64id') -const basicAuth = require('basic-auth-connect'); -const httpAuth = basicAuth(function(user, pass){ - const isValid = (user === settings.internal.realTime.user) && (pass === settings.internal.realTime.pass); - if (!isValid) { - logger.err({user, pass}, "invalid login details"); - } - return isValid; -}); +const basicAuth = require('basic-auth-connect') +const httpAuth = basicAuth(function (user, pass) { + const isValid = + user === settings.internal.realTime.user && + pass === settings.internal.realTime.pass + if (!isValid) { + logger.err({ user, pass }, 'invalid login details') + } + return isValid +}) -module.exports = (Router = { - _handleError(callback, error, client, method, attrs) { - if (callback == null) { callback = function(error) {}; } - if (attrs == null) { attrs = {}; } - for (const key of ["project_id", "doc_id", "user_id"]) { - attrs[key] = client.ol_context[key]; - } - attrs.client_id = client.id; - attrs.err = error; - if (error.name === "CodedError") { - logger.warn(attrs, error.message, {code: error.code}); - return callback({message: error.message, code: error.code}); - } - if (error.message === 'unexpected arguments') { - // the payload might be very large, put it on level info - logger.log(attrs, 'unexpected arguments'); - metrics.inc('unexpected-arguments', 1, { status: method }); - return callback({ message: error.message }); - } - if (["not authorized", "doc updater could not load requested ops", "no project_id found on client"].includes(error.message)) { - logger.warn(attrs, error.message); - return callback({message: error.message}); - } else { - logger.error(attrs, `server side error in ${method}`); - // Don't return raw error to prevent leaking server side info - return callback({message: "Something went wrong in real-time service"}); - } - }, +module.exports = Router = { + _handleError(callback, error, client, method, attrs) { + if (callback == null) { + callback = function (error) {} + } + if (attrs == null) { + attrs = {} + } + for (const key of ['project_id', 'doc_id', 'user_id']) { + attrs[key] = client.ol_context[key] + } + attrs.client_id = client.id + attrs.err = error + if (error.name === 'CodedError') { + logger.warn(attrs, error.message, { code: error.code }) + return callback({ message: error.message, code: error.code }) + } + if (error.message === 'unexpected arguments') { + // the payload might be very large, put it on level info + logger.log(attrs, 'unexpected arguments') + metrics.inc('unexpected-arguments', 1, { status: method }) + return callback({ message: error.message }) + } + if ( + [ + 'not authorized', + 'doc updater could not load requested ops', + 'no project_id found on client' + ].includes(error.message) + ) { + logger.warn(attrs, error.message) + return callback({ message: error.message }) + } else { + logger.error(attrs, `server side error in ${method}`) + // Don't return raw error to prevent leaking server side info + return callback({ message: 'Something went wrong in real-time service' }) + } + }, - _handleInvalidArguments(client, method, args) { - const error = new Error("unexpected arguments"); - let callback = args[args.length - 1]; - if (typeof callback !== 'function') { - callback = (function() {}); - } - const attrs = {arguments: args}; - return Router._handleError(callback, error, client, method, attrs); - }, + _handleInvalidArguments(client, method, args) { + const error = new Error('unexpected arguments') + let callback = args[args.length - 1] + if (typeof callback !== 'function') { + callback = function () {} + } + const attrs = { arguments: args } + return Router._handleError(callback, error, client, method, attrs) + }, - configure(app, io, session) { - app.set("io", io); - app.get("/clients", HttpController.getConnectedClients); - app.get("/clients/:client_id", HttpController.getConnectedClient); + configure(app, io, session) { + app.set('io', io) + app.get('/clients', HttpController.getConnectedClients) + app.get('/clients/:client_id', HttpController.getConnectedClient) - app.post("/project/:project_id/message/:message", httpAuth, bodyParser.json({limit: "5mb"}), HttpApiController.sendMessage); - - app.post("/drain", httpAuth, HttpApiController.startDrain); - app.post("/client/:client_id/disconnect", httpAuth, HttpApiController.disconnectClient); + app.post( + '/project/:project_id/message/:message', + httpAuth, + bodyParser.json({ limit: '5mb' }), + HttpApiController.sendMessage + ) - return session.on('connection', function(error, client, session) { - // init client context, we may access it in Router._handleError before - // setting any values - let user; - client.ol_context = {}; + app.post('/drain', httpAuth, HttpApiController.startDrain) + app.post( + '/client/:client_id/disconnect', + httpAuth, + HttpApiController.disconnectClient + ) - if (client != null) { - client.on("error", function(err) { - logger.err({ clientErr: err }, "socket.io client error"); - if (client.connected) { - client.emit("reconnectGracefully"); - return client.disconnect(); - } - }); - } + return session.on('connection', function (error, client, session) { + // init client context, we may access it in Router._handleError before + // setting any values + let user + client.ol_context = {} - if (settings.shutDownInProgress) { - client.emit("connectionRejected", {message: "retry"}); - client.disconnect(); - return; - } + if (client != null) { + client.on('error', function (err) { + logger.err({ clientErr: err }, 'socket.io client error') + if (client.connected) { + client.emit('reconnectGracefully') + return client.disconnect() + } + }) + } - if ((client != null) && __guard__(error != null ? error.message : undefined, x => x.match(/could not look up session by key/))) { - logger.warn({err: error, client: (client != null), session: (session != null)}, "invalid session"); - // tell the client to reauthenticate if it has an invalid session key - client.emit("connectionRejected", {message: "invalid session"}); - client.disconnect(); - return; - } + if (settings.shutDownInProgress) { + client.emit('connectionRejected', { message: 'retry' }) + client.disconnect() + return + } - if (error != null) { - logger.err({err: error, client: (client != null), session: (session != null)}, "error when client connected"); - if (client != null) { - client.emit("connectionRejected", {message: "error"}); - } - if (client != null) { - client.disconnect(); - } - return; - } + if ( + client != null && + __guard__(error != null ? error.message : undefined, (x) => + x.match(/could not look up session by key/) + ) + ) { + logger.warn( + { err: error, client: client != null, session: session != null }, + 'invalid session' + ) + // tell the client to reauthenticate if it has an invalid session key + client.emit('connectionRejected', { message: 'invalid session' }) + client.disconnect() + return + } - // send positive confirmation that the client has a valid connection - client.publicId = 'P.' + base64id.generateId(); - client.emit("connectionAccepted", null, client.publicId); + if (error != null) { + logger.err( + { err: error, client: client != null, session: session != null }, + 'error when client connected' + ) + if (client != null) { + client.emit('connectionRejected', { message: 'error' }) + } + if (client != null) { + client.disconnect() + } + return + } - metrics.inc('socket-io.connection'); - metrics.gauge('socket-io.clients', __guard__(io.sockets.clients(), x1 => x1.length)); + // send positive confirmation that the client has a valid connection + client.publicId = 'P.' + base64id.generateId() + client.emit('connectionAccepted', null, client.publicId) - logger.log({session, client_id: client.id}, "client connected"); + metrics.inc('socket-io.connection') + metrics.gauge( + 'socket-io.clients', + __guard__(io.sockets.clients(), (x1) => x1.length) + ) - if (__guard__(session != null ? session.passport : undefined, x2 => x2.user) != null) { - ({ - user - } = session.passport); - } else if ((session != null ? session.user : undefined) != null) { - ({ - user - } = session); - } else { - user = {_id: "anonymous-user"}; - } + logger.log({ session, client_id: client.id }, 'client connected') - client.on("joinProject", function(data, callback) { - if (data == null) { data = {}; } - if (typeof callback !== 'function') { - return Router._handleInvalidArguments(client, 'joinProject', arguments); - } + if ( + __guard__( + session != null ? session.passport : undefined, + (x2) => x2.user + ) != null + ) { + ;({ user } = session.passport) + } else if ((session != null ? session.user : undefined) != null) { + ;({ user } = session) + } else { + user = { _id: 'anonymous-user' } + } - if (data.anonymousAccessToken) { - user.anonymousAccessToken = data.anonymousAccessToken; - } - return WebsocketController.joinProject(client, user, data.project_id, function(err, ...args) { - if (err != null) { - return Router._handleError(callback, err, client, "joinProject", {project_id: data.project_id, user_id: (user != null ? user.id : undefined)}); - } else { - return callback(null, ...Array.from(args)); - } - }); - }); + client.on('joinProject', function (data, callback) { + if (data == null) { + data = {} + } + if (typeof callback !== 'function') { + return Router._handleInvalidArguments( + client, + 'joinProject', + arguments + ) + } - client.on("disconnect", function() { - metrics.inc('socket-io.disconnect'); - metrics.gauge('socket-io.clients', __guard__(io.sockets.clients(), x3 => x3.length) - 1); + if (data.anonymousAccessToken) { + user.anonymousAccessToken = data.anonymousAccessToken + } + return WebsocketController.joinProject( + client, + user, + data.project_id, + function (err, ...args) { + if (err != null) { + return Router._handleError(callback, err, client, 'joinProject', { + project_id: data.project_id, + user_id: user != null ? user.id : undefined + }) + } else { + return callback(null, ...Array.from(args)) + } + } + ) + }) - return WebsocketController.leaveProject(io, client, function(err) { - if (err != null) { - return Router._handleError((function() {}), err, client, "leaveProject"); - } - }); - }); + client.on('disconnect', function () { + metrics.inc('socket-io.disconnect') + metrics.gauge( + 'socket-io.clients', + __guard__(io.sockets.clients(), (x3) => x3.length) - 1 + ) - // Variadic. The possible arguments: - // doc_id, callback - // doc_id, fromVersion, callback - // doc_id, options, callback - // doc_id, fromVersion, options, callback - client.on("joinDoc", function(doc_id, fromVersion, options, callback) { - if ((typeof fromVersion === "function") && !options) { - callback = fromVersion; - fromVersion = -1; - options = {}; - } else if ((typeof fromVersion === "number") && (typeof options === "function")) { - callback = options; - options = {}; - } else if ((typeof fromVersion === "object") && (typeof options === "function")) { - callback = options; - options = fromVersion; - fromVersion = -1; - } else if ((typeof fromVersion === "number") && (typeof options === "object") && (typeof callback === 'function')) { - // Called with 4 args, things are as expected - } else { - return Router._handleInvalidArguments(client, 'joinDoc', arguments); - } + return WebsocketController.leaveProject(io, client, function (err) { + if (err != null) { + return Router._handleError( + function () {}, + err, + client, + 'leaveProject' + ) + } + }) + }) - return WebsocketController.joinDoc(client, doc_id, fromVersion, options, function(err, ...args) { - if (err != null) { - return Router._handleError(callback, err, client, "joinDoc", {doc_id, fromVersion}); - } else { - return callback(null, ...Array.from(args)); - } - }); - }); + // Variadic. The possible arguments: + // doc_id, callback + // doc_id, fromVersion, callback + // doc_id, options, callback + // doc_id, fromVersion, options, callback + client.on('joinDoc', function (doc_id, fromVersion, options, callback) { + if (typeof fromVersion === 'function' && !options) { + callback = fromVersion + fromVersion = -1 + options = {} + } else if ( + typeof fromVersion === 'number' && + typeof options === 'function' + ) { + callback = options + options = {} + } else if ( + typeof fromVersion === 'object' && + typeof options === 'function' + ) { + callback = options + options = fromVersion + fromVersion = -1 + } else if ( + typeof fromVersion === 'number' && + typeof options === 'object' && + typeof callback === 'function' + ) { + // Called with 4 args, things are as expected + } else { + return Router._handleInvalidArguments(client, 'joinDoc', arguments) + } - client.on("leaveDoc", function(doc_id, callback) { - if (typeof callback !== 'function') { - return Router._handleInvalidArguments(client, 'leaveDoc', arguments); - } + return WebsocketController.joinDoc( + client, + doc_id, + fromVersion, + options, + function (err, ...args) { + if (err != null) { + return Router._handleError(callback, err, client, 'joinDoc', { + doc_id, + fromVersion + }) + } else { + return callback(null, ...Array.from(args)) + } + } + ) + }) - return WebsocketController.leaveDoc(client, doc_id, function(err, ...args) { - if (err != null) { - return Router._handleError(callback, err, client, "leaveDoc"); - } else { - return callback(null, ...Array.from(args)); - } - }); - }); + client.on('leaveDoc', function (doc_id, callback) { + if (typeof callback !== 'function') { + return Router._handleInvalidArguments(client, 'leaveDoc', arguments) + } - client.on("clientTracking.getConnectedUsers", function(callback) { - if (callback == null) { callback = function(error, users) {}; } - if (typeof callback !== 'function') { - return Router._handleInvalidArguments(client, 'clientTracking.getConnectedUsers', arguments); - } + return WebsocketController.leaveDoc(client, doc_id, function ( + err, + ...args + ) { + if (err != null) { + return Router._handleError(callback, err, client, 'leaveDoc') + } else { + return callback(null, ...Array.from(args)) + } + }) + }) - return WebsocketController.getConnectedUsers(client, function(err, users) { - if (err != null) { - return Router._handleError(callback, err, client, "clientTracking.getConnectedUsers"); - } else { - return callback(null, users); - } - }); - }); + client.on('clientTracking.getConnectedUsers', function (callback) { + if (callback == null) { + callback = function (error, users) {} + } + if (typeof callback !== 'function') { + return Router._handleInvalidArguments( + client, + 'clientTracking.getConnectedUsers', + arguments + ) + } - client.on("clientTracking.updatePosition", function(cursorData, callback) { - if (callback == null) { callback = function(error) {}; } - if (typeof callback !== 'function') { - return Router._handleInvalidArguments(client, 'clientTracking.updatePosition', arguments); - } + return WebsocketController.getConnectedUsers(client, function ( + err, + users + ) { + if (err != null) { + return Router._handleError( + callback, + err, + client, + 'clientTracking.getConnectedUsers' + ) + } else { + return callback(null, users) + } + }) + }) - return WebsocketController.updateClientPosition(client, cursorData, function(err) { - if (err != null) { - return Router._handleError(callback, err, client, "clientTracking.updatePosition"); - } else { - return callback(); - } - }); - }); + client.on('clientTracking.updatePosition', function ( + cursorData, + callback + ) { + if (callback == null) { + callback = function (error) {} + } + if (typeof callback !== 'function') { + return Router._handleInvalidArguments( + client, + 'clientTracking.updatePosition', + arguments + ) + } - return client.on("applyOtUpdate", function(doc_id, update, callback) { - if (callback == null) { callback = function(error) {}; } - if (typeof callback !== 'function') { - return Router._handleInvalidArguments(client, 'applyOtUpdate', arguments); - } + return WebsocketController.updateClientPosition( + client, + cursorData, + function (err) { + if (err != null) { + return Router._handleError( + callback, + err, + client, + 'clientTracking.updatePosition' + ) + } else { + return callback() + } + } + ) + }) - return WebsocketController.applyOtUpdate(client, doc_id, update, function(err) { - if (err != null) { - return Router._handleError(callback, err, client, "applyOtUpdate", {doc_id, update}); - } else { - return callback(); - } - }); - }); - }); - } -}); + return client.on('applyOtUpdate', function (doc_id, update, callback) { + if (callback == null) { + callback = function (error) {} + } + if (typeof callback !== 'function') { + return Router._handleInvalidArguments( + client, + 'applyOtUpdate', + arguments + ) + } + + return WebsocketController.applyOtUpdate( + client, + doc_id, + update, + function (err) { + if (err != null) { + return Router._handleError( + callback, + err, + client, + 'applyOtUpdate', + { doc_id, update } + ) + } else { + return callback() + } + } + ) + }) + }) + } +} function __guard__(value, transform) { - return (typeof value !== 'undefined' && value !== null) ? transform(value) : undefined; -} \ No newline at end of file + return typeof value !== 'undefined' && value !== null + ? transform(value) + : undefined +} diff --git a/services/real-time/app/js/SafeJsonParse.js b/services/real-time/app/js/SafeJsonParse.js index f5e8dd3797..6e2e287853 100644 --- a/services/real-time/app/js/SafeJsonParse.js +++ b/services/real-time/app/js/SafeJsonParse.js @@ -9,22 +9,27 @@ * DS207: Consider shorter variations of null checks * Full docs: https://github.com/decaffeinate/decaffeinate/blob/master/docs/suggestions.md */ -const Settings = require("settings-sharelatex"); -const logger = require("logger-sharelatex"); +const Settings = require('settings-sharelatex') +const logger = require('logger-sharelatex') module.exports = { - parse(data, callback) { - let parsed; - if (callback == null) { callback = function(error, parsed) {}; } - if (data.length > Settings.maxUpdateSize) { - logger.error({head: data.slice(0,1024), length: data.length}, "data too large to parse"); - return callback(new Error("data too large to parse")); - } - try { - parsed = JSON.parse(data); - } catch (e) { - return callback(e); - } - return callback(null, parsed); - } -}; \ No newline at end of file + parse(data, callback) { + let parsed + if (callback == null) { + callback = function (error, parsed) {} + } + if (data.length > Settings.maxUpdateSize) { + logger.error( + { head: data.slice(0, 1024), length: data.length }, + 'data too large to parse' + ) + return callback(new Error('data too large to parse')) + } + try { + parsed = JSON.parse(data) + } catch (e) { + return callback(e) + } + return callback(null, parsed) + } +} diff --git a/services/real-time/app/js/SessionSockets.js b/services/real-time/app/js/SessionSockets.js index 894c7b53d5..b01920dfa7 100644 --- a/services/real-time/app/js/SessionSockets.js +++ b/services/real-time/app/js/SessionSockets.js @@ -5,32 +5,33 @@ * DS102: Remove unnecessary code created because of implicit returns * Full docs: https://github.com/decaffeinate/decaffeinate/blob/master/docs/suggestions.md */ -const {EventEmitter} = require('events'); +const { EventEmitter } = require('events') -module.exports = function(io, sessionStore, cookieParser, cookieName) { - const missingSessionError = new Error('could not look up session by key'); +module.exports = function (io, sessionStore, cookieParser, cookieName) { + const missingSessionError = new Error('could not look up session by key') - const sessionSockets = new EventEmitter(); - const next = (error, socket, session) => sessionSockets.emit('connection', error, socket, session); + const sessionSockets = new EventEmitter() + const next = (error, socket, session) => + sessionSockets.emit('connection', error, socket, session) - io.on('connection', function(socket) { - const req = socket.handshake; - return cookieParser(req, {}, function() { - const sessionId = req.signedCookies && req.signedCookies[cookieName]; - if (!sessionId) { - return next(missingSessionError, socket); - } - return sessionStore.get(sessionId, function(error, session) { - if (error) { - return next(error, socket); - } - if (!session) { - return next(missingSessionError, socket); - } - return next(null, socket, session); - }); - }); - }); + io.on('connection', function (socket) { + const req = socket.handshake + return cookieParser(req, {}, function () { + const sessionId = req.signedCookies && req.signedCookies[cookieName] + if (!sessionId) { + return next(missingSessionError, socket) + } + return sessionStore.get(sessionId, function (error, session) { + if (error) { + return next(error, socket) + } + if (!session) { + return next(missingSessionError, socket) + } + return next(null, socket, session) + }) + }) + }) - return sessionSockets; -}; + return sessionSockets +} diff --git a/services/real-time/app/js/WebApiManager.js b/services/real-time/app/js/WebApiManager.js index 9598d83106..266135333a 100644 --- a/services/real-time/app/js/WebApiManager.js +++ b/services/real-time/app/js/WebApiManager.js @@ -11,51 +11,76 @@ * DS207: Consider shorter variations of null checks * Full docs: https://github.com/decaffeinate/decaffeinate/blob/master/docs/suggestions.md */ -let WebApiManager; -const request = require("request"); -const settings = require("settings-sharelatex"); -const logger = require("logger-sharelatex"); -const { CodedError } = require("./Errors"); +let WebApiManager +const request = require('request') +const settings = require('settings-sharelatex') +const logger = require('logger-sharelatex') +const { CodedError } = require('./Errors') -module.exports = (WebApiManager = { - joinProject(project_id, user, callback) { - if (callback == null) { callback = function(error, project, privilegeLevel, isRestrictedUser) {}; } - const user_id = user._id; - logger.log({project_id, user_id}, "sending join project request to web"); - const url = `${settings.apis.web.url}/project/${project_id}/join`; - const headers = {}; - if (user.anonymousAccessToken != null) { - headers['x-sl-anonymous-access-token'] = user.anonymousAccessToken; - } - return request.post({ - url, - qs: {user_id}, - auth: { - user: settings.apis.web.user, - pass: settings.apis.web.pass, - sendImmediately: true - }, - json: true, - jar: false, - headers - }, function(error, response, data) { - let err; - if (error != null) { return callback(error); } - if (response.statusCode >= 200 && response.statusCode < 300) { - if ((data == null) || ((data != null ? data.project : undefined) == null)) { - err = new Error('no data returned from joinProject request'); - logger.error({err, project_id, user_id}, "error accessing web api"); - return callback(err); - } - return callback(null, data.project, data.privilegeLevel, data.isRestrictedUser); - } else if (response.statusCode === 429) { - logger.log(project_id, user_id, "rate-limit hit when joining project"); - return callback(new CodedError("rate-limit hit when joining project", "TooManyRequests")); - } else { - err = new Error(`non-success status code from web: ${response.statusCode}`); - logger.error({err, project_id, user_id}, "error accessing web api"); - return callback(err); - } - }); - } -}); +module.exports = WebApiManager = { + joinProject(project_id, user, callback) { + if (callback == null) { + callback = function (error, project, privilegeLevel, isRestrictedUser) {} + } + const user_id = user._id + logger.log({ project_id, user_id }, 'sending join project request to web') + const url = `${settings.apis.web.url}/project/${project_id}/join` + const headers = {} + if (user.anonymousAccessToken != null) { + headers['x-sl-anonymous-access-token'] = user.anonymousAccessToken + } + return request.post( + { + url, + qs: { user_id }, + auth: { + user: settings.apis.web.user, + pass: settings.apis.web.pass, + sendImmediately: true + }, + json: true, + jar: false, + headers + }, + function (error, response, data) { + let err + if (error != null) { + return callback(error) + } + if (response.statusCode >= 200 && response.statusCode < 300) { + if ( + data == null || + (data != null ? data.project : undefined) == null + ) { + err = new Error('no data returned from joinProject request') + logger.error( + { err, project_id, user_id }, + 'error accessing web api' + ) + return callback(err) + } + return callback( + null, + data.project, + data.privilegeLevel, + data.isRestrictedUser + ) + } else if (response.statusCode === 429) { + logger.log(project_id, user_id, 'rate-limit hit when joining project') + return callback( + new CodedError( + 'rate-limit hit when joining project', + 'TooManyRequests' + ) + ) + } else { + err = new Error( + `non-success status code from web: ${response.statusCode}` + ) + logger.error({ err, project_id, user_id }, 'error accessing web api') + return callback(err) + } + } + ) + } +} diff --git a/services/real-time/app/js/WebsocketController.js b/services/real-time/app/js/WebsocketController.js index aa51bbb372..92632cc154 100644 --- a/services/real-time/app/js/WebsocketController.js +++ b/services/real-time/app/js/WebsocketController.js @@ -13,344 +13,596 @@ * DS207: Consider shorter variations of null checks * Full docs: https://github.com/decaffeinate/decaffeinate/blob/master/docs/suggestions.md */ -let WebsocketController; -const logger = require("logger-sharelatex"); -const metrics = require("metrics-sharelatex"); -const settings = require("settings-sharelatex"); -const WebApiManager = require("./WebApiManager"); -const AuthorizationManager = require("./AuthorizationManager"); -const DocumentUpdaterManager = require("./DocumentUpdaterManager"); -const ConnectedUsersManager = require("./ConnectedUsersManager"); -const WebsocketLoadBalancer = require("./WebsocketLoadBalancer"); -const RoomManager = require("./RoomManager"); +let WebsocketController +const logger = require('logger-sharelatex') +const metrics = require('metrics-sharelatex') +const settings = require('settings-sharelatex') +const WebApiManager = require('./WebApiManager') +const AuthorizationManager = require('./AuthorizationManager') +const DocumentUpdaterManager = require('./DocumentUpdaterManager') +const ConnectedUsersManager = require('./ConnectedUsersManager') +const WebsocketLoadBalancer = require('./WebsocketLoadBalancer') +const RoomManager = require('./RoomManager') -module.exports = (WebsocketController = { - // If the protocol version changes when the client reconnects, - // it will force a full refresh of the page. Useful for non-backwards - // compatible protocol changes. Use only in extreme need. - PROTOCOL_VERSION: 2, +module.exports = WebsocketController = { + // If the protocol version changes when the client reconnects, + // it will force a full refresh of the page. Useful for non-backwards + // compatible protocol changes. Use only in extreme need. + PROTOCOL_VERSION: 2, - joinProject(client, user, project_id, callback) { - if (callback == null) { callback = function(error, project, privilegeLevel, protocolVersion) {}; } - if (client.disconnected) { - metrics.inc('editor.join-project.disconnected', 1, {status: 'immediately'}); - return callback(); - } + joinProject(client, user, project_id, callback) { + if (callback == null) { + callback = function (error, project, privilegeLevel, protocolVersion) {} + } + if (client.disconnected) { + metrics.inc('editor.join-project.disconnected', 1, { + status: 'immediately' + }) + return callback() + } - const user_id = user != null ? user._id : undefined; - logger.log({user_id, project_id, client_id: client.id}, "user joining project"); - metrics.inc("editor.join-project"); - return WebApiManager.joinProject(project_id, user, function(error, project, privilegeLevel, isRestrictedUser) { - if (error != null) { return callback(error); } - if (client.disconnected) { - metrics.inc('editor.join-project.disconnected', 1, {status: 'after-web-api-call'}); - return callback(); - } + const user_id = user != null ? user._id : undefined + logger.log( + { user_id, project_id, client_id: client.id }, + 'user joining project' + ) + metrics.inc('editor.join-project') + return WebApiManager.joinProject(project_id, user, function ( + error, + project, + privilegeLevel, + isRestrictedUser + ) { + if (error != null) { + return callback(error) + } + if (client.disconnected) { + metrics.inc('editor.join-project.disconnected', 1, { + status: 'after-web-api-call' + }) + return callback() + } - if (!privilegeLevel || (privilegeLevel === "")) { - const err = new Error("not authorized"); - logger.warn({err, project_id, user_id, client_id: client.id}, "user is not authorized to join project"); - return callback(err); - } + if (!privilegeLevel || privilegeLevel === '') { + const err = new Error('not authorized') + logger.warn( + { err, project_id, user_id, client_id: client.id }, + 'user is not authorized to join project' + ) + return callback(err) + } - client.ol_context = {}; - client.ol_context.privilege_level = privilegeLevel; - client.ol_context.user_id = user_id; - client.ol_context.project_id = project_id; - client.ol_context.owner_id = __guard__(project != null ? project.owner : undefined, x => x._id); - client.ol_context.first_name = user != null ? user.first_name : undefined; - client.ol_context.last_name = user != null ? user.last_name : undefined; - client.ol_context.email = user != null ? user.email : undefined; - client.ol_context.connected_time = new Date(); - client.ol_context.signup_date = user != null ? user.signUpDate : undefined; - client.ol_context.login_count = user != null ? user.loginCount : undefined; - client.ol_context.is_restricted_user = !!(isRestrictedUser); + client.ol_context = {} + client.ol_context.privilege_level = privilegeLevel + client.ol_context.user_id = user_id + client.ol_context.project_id = project_id + client.ol_context.owner_id = __guard__( + project != null ? project.owner : undefined, + (x) => x._id + ) + client.ol_context.first_name = user != null ? user.first_name : undefined + client.ol_context.last_name = user != null ? user.last_name : undefined + client.ol_context.email = user != null ? user.email : undefined + client.ol_context.connected_time = new Date() + client.ol_context.signup_date = user != null ? user.signUpDate : undefined + client.ol_context.login_count = user != null ? user.loginCount : undefined + client.ol_context.is_restricted_user = !!isRestrictedUser - RoomManager.joinProject(client, project_id, function(err) { - if (err) { return callback(err); } - logger.log({user_id, project_id, client_id: client.id}, "user joined project"); - return callback(null, project, privilegeLevel, WebsocketController.PROTOCOL_VERSION); - }); + RoomManager.joinProject(client, project_id, function (err) { + if (err) { + return callback(err) + } + logger.log( + { user_id, project_id, client_id: client.id }, + 'user joined project' + ) + return callback( + null, + project, + privilegeLevel, + WebsocketController.PROTOCOL_VERSION + ) + }) - // No need to block for setting the user as connected in the cursor tracking - return ConnectedUsersManager.updateUserPosition(project_id, client.publicId, user, null, function() {}); - }); - }, + // No need to block for setting the user as connected in the cursor tracking + return ConnectedUsersManager.updateUserPosition( + project_id, + client.publicId, + user, + null, + function () {} + ) + }) + }, - // We want to flush a project if there are no more (local) connected clients - // but we need to wait for the triggering client to disconnect. How long we wait - // is determined by FLUSH_IF_EMPTY_DELAY. - FLUSH_IF_EMPTY_DELAY: 500, // ms - leaveProject(io, client, callback) { - if (callback == null) { callback = function(error) {}; } - const {project_id, user_id} = client.ol_context; - if (!project_id) { return callback(); } // client did not join project + // We want to flush a project if there are no more (local) connected clients + // but we need to wait for the triggering client to disconnect. How long we wait + // is determined by FLUSH_IF_EMPTY_DELAY. + FLUSH_IF_EMPTY_DELAY: 500, // ms + leaveProject(io, client, callback) { + if (callback == null) { + callback = function (error) {} + } + const { project_id, user_id } = client.ol_context + if (!project_id) { + return callback() + } // client did not join project - metrics.inc("editor.leave-project"); - logger.log({project_id, user_id, client_id: client.id}, "client leaving project"); - WebsocketLoadBalancer.emitToRoom(project_id, "clientTracking.clientDisconnected", client.publicId); + metrics.inc('editor.leave-project') + logger.log( + { project_id, user_id, client_id: client.id }, + 'client leaving project' + ) + WebsocketLoadBalancer.emitToRoom( + project_id, + 'clientTracking.clientDisconnected', + client.publicId + ) - // We can do this in the background - ConnectedUsersManager.markUserAsDisconnected(project_id, client.publicId, function(err) { - if (err != null) { - return logger.error({err, project_id, user_id, client_id: client.id}, "error marking client as disconnected"); - } - }); + // We can do this in the background + ConnectedUsersManager.markUserAsDisconnected( + project_id, + client.publicId, + function (err) { + if (err != null) { + return logger.error( + { err, project_id, user_id, client_id: client.id }, + 'error marking client as disconnected' + ) + } + } + ) - RoomManager.leaveProjectAndDocs(client); - return setTimeout(function() { - const remainingClients = io.sockets.clients(project_id); - if (remainingClients.length === 0) { - // Flush project in the background - DocumentUpdaterManager.flushProjectToMongoAndDelete(project_id, function(err) { - if (err != null) { - return logger.error({err, project_id, user_id, client_id: client.id}, "error flushing to doc updater after leaving project"); - } - }); - } - return callback(); - } - , WebsocketController.FLUSH_IF_EMPTY_DELAY); - }, + RoomManager.leaveProjectAndDocs(client) + return setTimeout(function () { + const remainingClients = io.sockets.clients(project_id) + if (remainingClients.length === 0) { + // Flush project in the background + DocumentUpdaterManager.flushProjectToMongoAndDelete( + project_id, + function (err) { + if (err != null) { + return logger.error( + { err, project_id, user_id, client_id: client.id }, + 'error flushing to doc updater after leaving project' + ) + } + } + ) + } + return callback() + }, WebsocketController.FLUSH_IF_EMPTY_DELAY) + }, - joinDoc(client, doc_id, fromVersion, options, callback) { - if (fromVersion == null) { fromVersion = -1; } - if (callback == null) { callback = function(error, doclines, version, ops, ranges) {}; } - if (client.disconnected) { - metrics.inc('editor.join-doc.disconnected', 1, {status: 'immediately'}); - return callback(); - } + joinDoc(client, doc_id, fromVersion, options, callback) { + if (fromVersion == null) { + fromVersion = -1 + } + if (callback == null) { + callback = function (error, doclines, version, ops, ranges) {} + } + if (client.disconnected) { + metrics.inc('editor.join-doc.disconnected', 1, { status: 'immediately' }) + return callback() + } - metrics.inc("editor.join-doc"); - const {project_id, user_id, is_restricted_user} = client.ol_context; - if ((project_id == null)) { return callback(new Error("no project_id found on client")); } - logger.log({user_id, project_id, doc_id, fromVersion, client_id: client.id}, "client joining doc"); + metrics.inc('editor.join-doc') + const { project_id, user_id, is_restricted_user } = client.ol_context + if (project_id == null) { + return callback(new Error('no project_id found on client')) + } + logger.log( + { user_id, project_id, doc_id, fromVersion, client_id: client.id }, + 'client joining doc' + ) - return AuthorizationManager.assertClientCanViewProject(client, function(error) { - if (error != null) { return callback(error); } - // ensure the per-doc applied-ops channel is subscribed before sending the - // doc to the client, so that no events are missed. - return RoomManager.joinDoc(client, doc_id, function(error) { - if (error != null) { return callback(error); } - if (client.disconnected) { - metrics.inc('editor.join-doc.disconnected', 1, {status: 'after-joining-room'}); - // the client will not read the response anyways - return callback(); - } + return AuthorizationManager.assertClientCanViewProject(client, function ( + error + ) { + if (error != null) { + return callback(error) + } + // ensure the per-doc applied-ops channel is subscribed before sending the + // doc to the client, so that no events are missed. + return RoomManager.joinDoc(client, doc_id, function (error) { + if (error != null) { + return callback(error) + } + if (client.disconnected) { + metrics.inc('editor.join-doc.disconnected', 1, { + status: 'after-joining-room' + }) + // the client will not read the response anyways + return callback() + } - return DocumentUpdaterManager.getDocument(project_id, doc_id, fromVersion, function(error, lines, version, ranges, ops) { - let err; - if (error != null) { return callback(error); } - if (client.disconnected) { - metrics.inc('editor.join-doc.disconnected', 1, {status: 'after-doc-updater-call'}); - // the client will not read the response anyways - return callback(); - } + return DocumentUpdaterManager.getDocument( + project_id, + doc_id, + fromVersion, + function (error, lines, version, ranges, ops) { + let err + if (error != null) { + return callback(error) + } + if (client.disconnected) { + metrics.inc('editor.join-doc.disconnected', 1, { + status: 'after-doc-updater-call' + }) + // the client will not read the response anyways + return callback() + } - if (is_restricted_user && ((ranges != null ? ranges.comments : undefined) != null)) { - ranges.comments = []; - } + if ( + is_restricted_user && + (ranges != null ? ranges.comments : undefined) != null + ) { + ranges.comments = [] + } - // Encode any binary bits of data so it can go via WebSockets - // See http://ecmanaut.blogspot.co.uk/2006/07/encoding-decoding-utf8-in-javascript.html - const encodeForWebsockets = text => unescape(encodeURIComponent(text)); - const escapedLines = []; - for (let line of Array.from(lines)) { - try { - line = encodeForWebsockets(line); - } catch (error1) { - err = error1; - logger.err({err, project_id, doc_id, fromVersion, line, client_id: client.id}, "error encoding line uri component"); - return callback(err); - } - escapedLines.push(line); - } - if (options.encodeRanges) { - try { - for (const comment of Array.from((ranges != null ? ranges.comments : undefined) || [])) { - if (comment.op.c != null) { comment.op.c = encodeForWebsockets(comment.op.c); } - } - for (const change of Array.from((ranges != null ? ranges.changes : undefined) || [])) { - if (change.op.i != null) { change.op.i = encodeForWebsockets(change.op.i); } - if (change.op.d != null) { change.op.d = encodeForWebsockets(change.op.d); } - } - } catch (error2) { - err = error2; - logger.err({err, project_id, doc_id, fromVersion, ranges, client_id: client.id}, "error encoding range uri component"); - return callback(err); - } - } + // Encode any binary bits of data so it can go via WebSockets + // See http://ecmanaut.blogspot.co.uk/2006/07/encoding-decoding-utf8-in-javascript.html + const encodeForWebsockets = (text) => + unescape(encodeURIComponent(text)) + const escapedLines = [] + for (let line of Array.from(lines)) { + try { + line = encodeForWebsockets(line) + } catch (error1) { + err = error1 + logger.err( + { + err, + project_id, + doc_id, + fromVersion, + line, + client_id: client.id + }, + 'error encoding line uri component' + ) + return callback(err) + } + escapedLines.push(line) + } + if (options.encodeRanges) { + try { + for (const comment of Array.from( + (ranges != null ? ranges.comments : undefined) || [] + )) { + if (comment.op.c != null) { + comment.op.c = encodeForWebsockets(comment.op.c) + } + } + for (const change of Array.from( + (ranges != null ? ranges.changes : undefined) || [] + )) { + if (change.op.i != null) { + change.op.i = encodeForWebsockets(change.op.i) + } + if (change.op.d != null) { + change.op.d = encodeForWebsockets(change.op.d) + } + } + } catch (error2) { + err = error2 + logger.err( + { + err, + project_id, + doc_id, + fromVersion, + ranges, + client_id: client.id + }, + 'error encoding range uri component' + ) + return callback(err) + } + } - AuthorizationManager.addAccessToDoc(client, doc_id); - logger.log({user_id, project_id, doc_id, fromVersion, client_id: client.id}, "client joined doc"); - return callback(null, escapedLines, version, ops, ranges); - }); - }); - }); - }, + AuthorizationManager.addAccessToDoc(client, doc_id) + logger.log( + { + user_id, + project_id, + doc_id, + fromVersion, + client_id: client.id + }, + 'client joined doc' + ) + return callback(null, escapedLines, version, ops, ranges) + } + ) + }) + }) + }, - leaveDoc(client, doc_id, callback) { - // client may have disconnected, but we have to cleanup internal state. - if (callback == null) { callback = function(error) {}; } - metrics.inc("editor.leave-doc"); - const {project_id, user_id} = client.ol_context; - logger.log({user_id, project_id, doc_id, client_id: client.id}, "client leaving doc"); - RoomManager.leaveDoc(client, doc_id); - // we could remove permission when user leaves a doc, but because - // the connection is per-project, we continue to allow access - // after the initial joinDoc since we know they are already authorised. - // # AuthorizationManager.removeAccessToDoc client, doc_id - return callback(); - }, - updateClientPosition(client, cursorData, callback) { - if (callback == null) { callback = function(error) {}; } - if (client.disconnected) { - // do not create a ghost entry in redis - return callback(); - } + leaveDoc(client, doc_id, callback) { + // client may have disconnected, but we have to cleanup internal state. + if (callback == null) { + callback = function (error) {} + } + metrics.inc('editor.leave-doc') + const { project_id, user_id } = client.ol_context + logger.log( + { user_id, project_id, doc_id, client_id: client.id }, + 'client leaving doc' + ) + RoomManager.leaveDoc(client, doc_id) + // we could remove permission when user leaves a doc, but because + // the connection is per-project, we continue to allow access + // after the initial joinDoc since we know they are already authorised. + // # AuthorizationManager.removeAccessToDoc client, doc_id + return callback() + }, + updateClientPosition(client, cursorData, callback) { + if (callback == null) { + callback = function (error) {} + } + if (client.disconnected) { + // do not create a ghost entry in redis + return callback() + } - metrics.inc("editor.update-client-position", 0.1); - const {project_id, first_name, last_name, email, user_id} = client.ol_context; - logger.log({user_id, project_id, client_id: client.id, cursorData}, "updating client position"); + metrics.inc('editor.update-client-position', 0.1) + const { + project_id, + first_name, + last_name, + email, + user_id + } = client.ol_context + logger.log( + { user_id, project_id, client_id: client.id, cursorData }, + 'updating client position' + ) - return AuthorizationManager.assertClientCanViewProjectAndDoc(client, cursorData.doc_id, function(error) { - if (error != null) { - logger.warn({err: error, client_id: client.id, project_id, user_id}, "silently ignoring unauthorized updateClientPosition. Client likely hasn't called joinProject yet."); - return callback(); - } - cursorData.id = client.publicId; - if (user_id != null) { cursorData.user_id = user_id; } - if (email != null) { cursorData.email = email; } - // Don't store anonymous users in redis to avoid influx - if (!user_id || (user_id === 'anonymous-user')) { - cursorData.name = ""; - callback(); - } else { - cursorData.name = first_name && last_name ? - `${first_name} ${last_name}` - : first_name || (last_name || ""); - ConnectedUsersManager.updateUserPosition(project_id, client.publicId, { - first_name, - last_name, - email, - _id: user_id - }, { - row: cursorData.row, - column: cursorData.column, - doc_id: cursorData.doc_id - }, callback); - } - return WebsocketLoadBalancer.emitToRoom(project_id, "clientTracking.clientUpdated", cursorData); - }); - }, + return AuthorizationManager.assertClientCanViewProjectAndDoc( + client, + cursorData.doc_id, + function (error) { + if (error != null) { + logger.warn( + { err: error, client_id: client.id, project_id, user_id }, + "silently ignoring unauthorized updateClientPosition. Client likely hasn't called joinProject yet." + ) + return callback() + } + cursorData.id = client.publicId + if (user_id != null) { + cursorData.user_id = user_id + } + if (email != null) { + cursorData.email = email + } + // Don't store anonymous users in redis to avoid influx + if (!user_id || user_id === 'anonymous-user') { + cursorData.name = '' + callback() + } else { + cursorData.name = + first_name && last_name + ? `${first_name} ${last_name}` + : first_name || last_name || '' + ConnectedUsersManager.updateUserPosition( + project_id, + client.publicId, + { + first_name, + last_name, + email, + _id: user_id + }, + { + row: cursorData.row, + column: cursorData.column, + doc_id: cursorData.doc_id + }, + callback + ) + } + return WebsocketLoadBalancer.emitToRoom( + project_id, + 'clientTracking.clientUpdated', + cursorData + ) + } + ) + }, - CLIENT_REFRESH_DELAY: 1000, - getConnectedUsers(client, callback) { - if (callback == null) { callback = function(error, users) {}; } - if (client.disconnected) { - // they are not interested anymore, skip the redis lookups - return callback(); - } + CLIENT_REFRESH_DELAY: 1000, + getConnectedUsers(client, callback) { + if (callback == null) { + callback = function (error, users) {} + } + if (client.disconnected) { + // they are not interested anymore, skip the redis lookups + return callback() + } - metrics.inc("editor.get-connected-users"); - const {project_id, user_id, is_restricted_user} = client.ol_context; - if (is_restricted_user) { - return callback(null, []); - } - if ((project_id == null)) { return callback(new Error("no project_id found on client")); } - logger.log({user_id, project_id, client_id: client.id}, "getting connected users"); - return AuthorizationManager.assertClientCanViewProject(client, function(error) { - if (error != null) { return callback(error); } - WebsocketLoadBalancer.emitToRoom(project_id, 'clientTracking.refresh'); - return setTimeout(() => ConnectedUsersManager.getConnectedUsers(project_id, function(error, users) { - if (error != null) { return callback(error); } - callback(null, users); - return logger.log({user_id, project_id, client_id: client.id}, "got connected users"); - }) - , WebsocketController.CLIENT_REFRESH_DELAY); - }); - }, + metrics.inc('editor.get-connected-users') + const { project_id, user_id, is_restricted_user } = client.ol_context + if (is_restricted_user) { + return callback(null, []) + } + if (project_id == null) { + return callback(new Error('no project_id found on client')) + } + logger.log( + { user_id, project_id, client_id: client.id }, + 'getting connected users' + ) + return AuthorizationManager.assertClientCanViewProject(client, function ( + error + ) { + if (error != null) { + return callback(error) + } + WebsocketLoadBalancer.emitToRoom(project_id, 'clientTracking.refresh') + return setTimeout( + () => + ConnectedUsersManager.getConnectedUsers(project_id, function ( + error, + users + ) { + if (error != null) { + return callback(error) + } + callback(null, users) + return logger.log( + { user_id, project_id, client_id: client.id }, + 'got connected users' + ) + }), + WebsocketController.CLIENT_REFRESH_DELAY + ) + }) + }, - applyOtUpdate(client, doc_id, update, callback) { - // client may have disconnected, but we can submit their update to doc-updater anyways. - if (callback == null) { callback = function(error) {}; } - const {user_id, project_id} = client.ol_context; - if ((project_id == null)) { return callback(new Error("no project_id found on client")); } + applyOtUpdate(client, doc_id, update, callback) { + // client may have disconnected, but we can submit their update to doc-updater anyways. + if (callback == null) { + callback = function (error) {} + } + const { user_id, project_id } = client.ol_context + if (project_id == null) { + return callback(new Error('no project_id found on client')) + } - return WebsocketController._assertClientCanApplyUpdate(client, doc_id, update, function(error) { - if (error != null) { - logger.warn({err: error, doc_id, client_id: client.id, version: update.v}, "client is not authorized to make update"); - setTimeout(() => // Disconnect, but give the client the chance to receive the error - client.disconnect() - , 100); - return callback(error); - } - if (!update.meta) { update.meta = {}; } - update.meta.source = client.publicId; - update.meta.user_id = user_id; - metrics.inc("editor.doc-update", 0.3); + return WebsocketController._assertClientCanApplyUpdate( + client, + doc_id, + update, + function (error) { + if (error != null) { + logger.warn( + { err: error, doc_id, client_id: client.id, version: update.v }, + 'client is not authorized to make update' + ) + setTimeout( + () => + // Disconnect, but give the client the chance to receive the error + client.disconnect(), + 100 + ) + return callback(error) + } + if (!update.meta) { + update.meta = {} + } + update.meta.source = client.publicId + update.meta.user_id = user_id + metrics.inc('editor.doc-update', 0.3) - logger.log({user_id, doc_id, project_id, client_id: client.id, version: update.v}, "sending update to doc updater"); + logger.log( + { + user_id, + doc_id, + project_id, + client_id: client.id, + version: update.v + }, + 'sending update to doc updater' + ) - return DocumentUpdaterManager.queueChange(project_id, doc_id, update, function(error) { - if ((error != null ? error.message : undefined) === "update is too large") { - metrics.inc("update_too_large"); - const { - updateSize - } = error; - logger.warn({user_id, project_id, doc_id, updateSize}, "update is too large"); + return DocumentUpdaterManager.queueChange( + project_id, + doc_id, + update, + function (error) { + if ( + (error != null ? error.message : undefined) === + 'update is too large' + ) { + metrics.inc('update_too_large') + const { updateSize } = error + logger.warn( + { user_id, project_id, doc_id, updateSize }, + 'update is too large' + ) - // mark the update as received -- the client should not send it again! - callback(); + // mark the update as received -- the client should not send it again! + callback() - // trigger an out-of-sync error - const message = {project_id, doc_id, error: "update is too large"}; - setTimeout(function() { - if (client.disconnected) { - // skip the message broadcast, the client has moved on - return metrics.inc('editor.doc-update.disconnected', 1, {status:'at-otUpdateError'}); - } - client.emit("otUpdateError", message.error, message); - return client.disconnect(); - } - , 100); - return; - } + // trigger an out-of-sync error + const message = { + project_id, + doc_id, + error: 'update is too large' + } + setTimeout(function () { + if (client.disconnected) { + // skip the message broadcast, the client has moved on + return metrics.inc('editor.doc-update.disconnected', 1, { + status: 'at-otUpdateError' + }) + } + client.emit('otUpdateError', message.error, message) + return client.disconnect() + }, 100) + return + } - if (error != null) { - logger.error({err: error, project_id, doc_id, client_id: client.id, version: update.v}, "document was not available for update"); - client.disconnect(); - } - return callback(error); - }); - }); - }, + if (error != null) { + logger.error( + { + err: error, + project_id, + doc_id, + client_id: client.id, + version: update.v + }, + 'document was not available for update' + ) + client.disconnect() + } + return callback(error) + } + ) + } + ) + }, - _assertClientCanApplyUpdate(client, doc_id, update, callback) { - return AuthorizationManager.assertClientCanEditProjectAndDoc(client, doc_id, function(error) { - if (error != null) { - if ((error.message === "not authorized") && WebsocketController._isCommentUpdate(update)) { - // This might be a comment op, which we only need read-only priveleges for - return AuthorizationManager.assertClientCanViewProjectAndDoc(client, doc_id, callback); - } else { - return callback(error); - } - } else { - return callback(null); - } - }); - }, + _assertClientCanApplyUpdate(client, doc_id, update, callback) { + return AuthorizationManager.assertClientCanEditProjectAndDoc( + client, + doc_id, + function (error) { + if (error != null) { + if ( + error.message === 'not authorized' && + WebsocketController._isCommentUpdate(update) + ) { + // This might be a comment op, which we only need read-only priveleges for + return AuthorizationManager.assertClientCanViewProjectAndDoc( + client, + doc_id, + callback + ) + } else { + return callback(error) + } + } else { + return callback(null) + } + } + ) + }, - _isCommentUpdate(update) { - for (const op of Array.from(update.op)) { - if ((op.c == null)) { - return false; - } - } - return true; - } -}); + _isCommentUpdate(update) { + for (const op of Array.from(update.op)) { + if (op.c == null) { + return false + } + } + return true + } +} function __guard__(value, transform) { - return (typeof value !== 'undefined' && value !== null) ? transform(value) : undefined; -} \ No newline at end of file + return typeof value !== 'undefined' && value !== null + ? transform(value) + : undefined +} diff --git a/services/real-time/app/js/WebsocketLoadBalancer.js b/services/real-time/app/js/WebsocketLoadBalancer.js index dc2617742a..2719921f10 100644 --- a/services/real-time/app/js/WebsocketLoadBalancer.js +++ b/services/real-time/app/js/WebsocketLoadBalancer.js @@ -11,146 +11,207 @@ * DS207: Consider shorter variations of null checks * Full docs: https://github.com/decaffeinate/decaffeinate/blob/master/docs/suggestions.md */ -let WebsocketLoadBalancer; -const Settings = require('settings-sharelatex'); -const logger = require('logger-sharelatex'); -const RedisClientManager = require("./RedisClientManager"); -const SafeJsonParse = require("./SafeJsonParse"); -const EventLogger = require("./EventLogger"); -const HealthCheckManager = require("./HealthCheckManager"); -const RoomManager = require("./RoomManager"); -const ChannelManager = require("./ChannelManager"); -const ConnectedUsersManager = require("./ConnectedUsersManager"); +let WebsocketLoadBalancer +const Settings = require('settings-sharelatex') +const logger = require('logger-sharelatex') +const RedisClientManager = require('./RedisClientManager') +const SafeJsonParse = require('./SafeJsonParse') +const EventLogger = require('./EventLogger') +const HealthCheckManager = require('./HealthCheckManager') +const RoomManager = require('./RoomManager') +const ChannelManager = require('./ChannelManager') +const ConnectedUsersManager = require('./ConnectedUsersManager') const RESTRICTED_USER_MESSAGE_TYPE_PASS_LIST = [ - 'connectionAccepted', - 'otUpdateApplied', - 'otUpdateError', - 'joinDoc', - 'reciveNewDoc', - 'reciveNewFile', - 'reciveNewFolder', - 'removeEntity' -]; + 'connectionAccepted', + 'otUpdateApplied', + 'otUpdateError', + 'joinDoc', + 'reciveNewDoc', + 'reciveNewFile', + 'reciveNewFolder', + 'removeEntity' +] -module.exports = (WebsocketLoadBalancer = { - rclientPubList: RedisClientManager.createClientList(Settings.redis.pubsub), - rclientSubList: RedisClientManager.createClientList(Settings.redis.pubsub), +module.exports = WebsocketLoadBalancer = { + rclientPubList: RedisClientManager.createClientList(Settings.redis.pubsub), + rclientSubList: RedisClientManager.createClientList(Settings.redis.pubsub), - emitToRoom(room_id, message, ...payload) { - if ((room_id == null)) { - logger.warn({message, payload}, "no room_id provided, ignoring emitToRoom"); - return; - } - const data = JSON.stringify({ - room_id, - message, - payload - }); - logger.log({room_id, message, payload, length: data.length}, "emitting to room"); + emitToRoom(room_id, message, ...payload) { + if (room_id == null) { + logger.warn( + { message, payload }, + 'no room_id provided, ignoring emitToRoom' + ) + return + } + const data = JSON.stringify({ + room_id, + message, + payload + }) + logger.log( + { room_id, message, payload, length: data.length }, + 'emitting to room' + ) - return Array.from(this.rclientPubList).map((rclientPub) => - ChannelManager.publish(rclientPub, "editor-events", room_id, data)); - }, + return Array.from(this.rclientPubList).map((rclientPub) => + ChannelManager.publish(rclientPub, 'editor-events', room_id, data) + ) + }, - emitToAll(message, ...payload) { - return this.emitToRoom("all", message, ...Array.from(payload)); - }, + emitToAll(message, ...payload) { + return this.emitToRoom('all', message, ...Array.from(payload)) + }, - listenForEditorEvents(io) { - logger.log({rclients: this.rclientPubList.length}, "publishing editor events"); - logger.log({rclients: this.rclientSubList.length}, "listening for editor events"); - for (const rclientSub of Array.from(this.rclientSubList)) { - rclientSub.subscribe("editor-events"); - rclientSub.on("message", function(channel, message) { - if (Settings.debugEvents > 0) { EventLogger.debugEvent(channel, message); } - return WebsocketLoadBalancer._processEditorEvent(io, channel, message); - }); - } - return this.handleRoomUpdates(this.rclientSubList); - }, + listenForEditorEvents(io) { + logger.log( + { rclients: this.rclientPubList.length }, + 'publishing editor events' + ) + logger.log( + { rclients: this.rclientSubList.length }, + 'listening for editor events' + ) + for (const rclientSub of Array.from(this.rclientSubList)) { + rclientSub.subscribe('editor-events') + rclientSub.on('message', function (channel, message) { + if (Settings.debugEvents > 0) { + EventLogger.debugEvent(channel, message) + } + return WebsocketLoadBalancer._processEditorEvent(io, channel, message) + }) + } + return this.handleRoomUpdates(this.rclientSubList) + }, - handleRoomUpdates(rclientSubList) { - const roomEvents = RoomManager.eventSource(); - roomEvents.on('project-active', function(project_id) { - const subscribePromises = Array.from(rclientSubList).map((rclient) => - ChannelManager.subscribe(rclient, "editor-events", project_id)); - return RoomManager.emitOnCompletion(subscribePromises, `project-subscribed-${project_id}`); - }); - return roomEvents.on('project-empty', project_id => Array.from(rclientSubList).map((rclient) => - ChannelManager.unsubscribe(rclient, "editor-events", project_id))); - }, + handleRoomUpdates(rclientSubList) { + const roomEvents = RoomManager.eventSource() + roomEvents.on('project-active', function (project_id) { + const subscribePromises = Array.from(rclientSubList).map((rclient) => + ChannelManager.subscribe(rclient, 'editor-events', project_id) + ) + return RoomManager.emitOnCompletion( + subscribePromises, + `project-subscribed-${project_id}` + ) + }) + return roomEvents.on('project-empty', (project_id) => + Array.from(rclientSubList).map((rclient) => + ChannelManager.unsubscribe(rclient, 'editor-events', project_id) + ) + ) + }, - _processEditorEvent(io, channel, message) { - return SafeJsonParse.parse(message, function(error, message) { - let clientList; - let client; - if (error != null) { - logger.error({err: error, channel}, "error parsing JSON"); - return; - } - if (message.room_id === "all") { - return io.sockets.emit(message.message, ...Array.from(message.payload)); - } else if ((message.message === 'clientTracking.refresh') && (message.room_id != null)) { - clientList = io.sockets.clients(message.room_id); - logger.log({channel, message: message.message, room_id: message.room_id, message_id: message._id, socketIoClients: ((() => { - const result = []; - for (client of Array.from(clientList)) { result.push(client.id); - } - return result; - })())}, "refreshing client list"); - return (() => { - const result1 = []; - for (client of Array.from(clientList)) { - result1.push(ConnectedUsersManager.refreshClient(message.room_id, client.publicId)); - } - return result1; - })(); - } else if (message.room_id != null) { - if ((message._id != null) && Settings.checkEventOrder) { - const status = EventLogger.checkEventOrder("editor-events", message._id, message); - if (status === "duplicate") { - return; // skip duplicate events - } - } + _processEditorEvent(io, channel, message) { + return SafeJsonParse.parse(message, function (error, message) { + let clientList + let client + if (error != null) { + logger.error({ err: error, channel }, 'error parsing JSON') + return + } + if (message.room_id === 'all') { + return io.sockets.emit(message.message, ...Array.from(message.payload)) + } else if ( + message.message === 'clientTracking.refresh' && + message.room_id != null + ) { + clientList = io.sockets.clients(message.room_id) + logger.log( + { + channel, + message: message.message, + room_id: message.room_id, + message_id: message._id, + socketIoClients: (() => { + const result = [] + for (client of Array.from(clientList)) { + result.push(client.id) + } + return result + })() + }, + 'refreshing client list' + ) + return (() => { + const result1 = [] + for (client of Array.from(clientList)) { + result1.push( + ConnectedUsersManager.refreshClient( + message.room_id, + client.publicId + ) + ) + } + return result1 + })() + } else if (message.room_id != null) { + if (message._id != null && Settings.checkEventOrder) { + const status = EventLogger.checkEventOrder( + 'editor-events', + message._id, + message + ) + if (status === 'duplicate') { + return // skip duplicate events + } + } - const is_restricted_message = !Array.from(RESTRICTED_USER_MESSAGE_TYPE_PASS_LIST).includes(message.message); + const is_restricted_message = !Array.from( + RESTRICTED_USER_MESSAGE_TYPE_PASS_LIST + ).includes(message.message) - // send messages only to unique clients (due to duplicate entries in io.sockets.clients) - clientList = io.sockets.clients(message.room_id) - .filter(client => !(is_restricted_message && client.ol_context.is_restricted_user)); + // send messages only to unique clients (due to duplicate entries in io.sockets.clients) + clientList = io.sockets + .clients(message.room_id) + .filter( + (client) => + !(is_restricted_message && client.ol_context.is_restricted_user) + ) - // avoid unnecessary work if no clients are connected - if (clientList.length === 0) { return; } - logger.log({ - channel, - message: message.message, - room_id: message.room_id, - message_id: message._id, - socketIoClients: ((() => { - const result2 = []; - for (client of Array.from(clientList)) { result2.push(client.id); - } - return result2; - })()) - }, "distributing event to clients"); - const seen = {}; - return (() => { - const result3 = []; - for (client of Array.from(clientList)) { - if (!seen[client.id]) { - seen[client.id] = true; - result3.push(client.emit(message.message, ...Array.from(message.payload))); - } else { - result3.push(undefined); - } - } - return result3; - })(); - } else if (message.health_check != null) { - logger.debug({message}, "got health check message in editor events channel"); - return HealthCheckManager.check(channel, message.key); - } - }); - } -}); + // avoid unnecessary work if no clients are connected + if (clientList.length === 0) { + return + } + logger.log( + { + channel, + message: message.message, + room_id: message.room_id, + message_id: message._id, + socketIoClients: (() => { + const result2 = [] + for (client of Array.from(clientList)) { + result2.push(client.id) + } + return result2 + })() + }, + 'distributing event to clients' + ) + const seen = {} + return (() => { + const result3 = [] + for (client of Array.from(clientList)) { + if (!seen[client.id]) { + seen[client.id] = true + result3.push( + client.emit(message.message, ...Array.from(message.payload)) + ) + } else { + result3.push(undefined) + } + } + return result3 + })() + } else if (message.health_check != null) { + logger.debug( + { message }, + 'got health check message in editor events channel' + ) + return HealthCheckManager.check(channel, message.key) + } + }) + } +} From e5d07bd3afbbccf5003e1adeedb97279e931555f Mon Sep 17 00:00:00 2001 From: decaffeinate Date: Tue, 23 Jun 2020 18:29:53 +0100 Subject: [PATCH 373/491] decaffeinate: Rename AuthorizationManagerTests.coffee and 13 other files from .coffee to .js --- ...horizationManagerTests.coffee => AuthorizationManagerTests.js} | 0 .../coffee/{ChannelManagerTests.coffee => ChannelManagerTests.js} | 0 ...ctedUsersManagerTests.coffee => ConnectedUsersManagerTests.js} | 0 ...erControllerTests.coffee => DocumentUpdaterControllerTests.js} | 0 ...tUpdaterManagerTests.coffee => DocumentUpdaterManagerTests.js} | 0 .../coffee/{DrainManagerTests.coffee => DrainManagerTests.js} | 0 .../unit/coffee/{EventLoggerTests.coffee => EventLoggerTests.js} | 0 .../unit/coffee/{RoomManagerTests.coffee => RoomManagerTests.js} | 0 .../coffee/{SafeJsonParseTest.coffee => SafeJsonParseTest.js} | 0 .../coffee/{SessionSocketsTests.coffee => SessionSocketsTests.js} | 0 .../coffee/{WebApiManagerTests.coffee => WebApiManagerTests.js} | 0 ...ebsocketControllerTests.coffee => WebsocketControllerTests.js} | 0 ...cketLoadBalancerTests.coffee => WebsocketLoadBalancerTests.js} | 0 .../test/unit/coffee/helpers/{MockClient.coffee => MockClient.js} | 0 14 files changed, 0 insertions(+), 0 deletions(-) rename services/real-time/test/unit/coffee/{AuthorizationManagerTests.coffee => AuthorizationManagerTests.js} (100%) rename services/real-time/test/unit/coffee/{ChannelManagerTests.coffee => ChannelManagerTests.js} (100%) rename services/real-time/test/unit/coffee/{ConnectedUsersManagerTests.coffee => ConnectedUsersManagerTests.js} (100%) rename services/real-time/test/unit/coffee/{DocumentUpdaterControllerTests.coffee => DocumentUpdaterControllerTests.js} (100%) rename services/real-time/test/unit/coffee/{DocumentUpdaterManagerTests.coffee => DocumentUpdaterManagerTests.js} (100%) rename services/real-time/test/unit/coffee/{DrainManagerTests.coffee => DrainManagerTests.js} (100%) rename services/real-time/test/unit/coffee/{EventLoggerTests.coffee => EventLoggerTests.js} (100%) rename services/real-time/test/unit/coffee/{RoomManagerTests.coffee => RoomManagerTests.js} (100%) rename services/real-time/test/unit/coffee/{SafeJsonParseTest.coffee => SafeJsonParseTest.js} (100%) rename services/real-time/test/unit/coffee/{SessionSocketsTests.coffee => SessionSocketsTests.js} (100%) rename services/real-time/test/unit/coffee/{WebApiManagerTests.coffee => WebApiManagerTests.js} (100%) rename services/real-time/test/unit/coffee/{WebsocketControllerTests.coffee => WebsocketControllerTests.js} (100%) rename services/real-time/test/unit/coffee/{WebsocketLoadBalancerTests.coffee => WebsocketLoadBalancerTests.js} (100%) rename services/real-time/test/unit/coffee/helpers/{MockClient.coffee => MockClient.js} (100%) diff --git a/services/real-time/test/unit/coffee/AuthorizationManagerTests.coffee b/services/real-time/test/unit/coffee/AuthorizationManagerTests.js similarity index 100% rename from services/real-time/test/unit/coffee/AuthorizationManagerTests.coffee rename to services/real-time/test/unit/coffee/AuthorizationManagerTests.js diff --git a/services/real-time/test/unit/coffee/ChannelManagerTests.coffee b/services/real-time/test/unit/coffee/ChannelManagerTests.js similarity index 100% rename from services/real-time/test/unit/coffee/ChannelManagerTests.coffee rename to services/real-time/test/unit/coffee/ChannelManagerTests.js diff --git a/services/real-time/test/unit/coffee/ConnectedUsersManagerTests.coffee b/services/real-time/test/unit/coffee/ConnectedUsersManagerTests.js similarity index 100% rename from services/real-time/test/unit/coffee/ConnectedUsersManagerTests.coffee rename to services/real-time/test/unit/coffee/ConnectedUsersManagerTests.js diff --git a/services/real-time/test/unit/coffee/DocumentUpdaterControllerTests.coffee b/services/real-time/test/unit/coffee/DocumentUpdaterControllerTests.js similarity index 100% rename from services/real-time/test/unit/coffee/DocumentUpdaterControllerTests.coffee rename to services/real-time/test/unit/coffee/DocumentUpdaterControllerTests.js diff --git a/services/real-time/test/unit/coffee/DocumentUpdaterManagerTests.coffee b/services/real-time/test/unit/coffee/DocumentUpdaterManagerTests.js similarity index 100% rename from services/real-time/test/unit/coffee/DocumentUpdaterManagerTests.coffee rename to services/real-time/test/unit/coffee/DocumentUpdaterManagerTests.js diff --git a/services/real-time/test/unit/coffee/DrainManagerTests.coffee b/services/real-time/test/unit/coffee/DrainManagerTests.js similarity index 100% rename from services/real-time/test/unit/coffee/DrainManagerTests.coffee rename to services/real-time/test/unit/coffee/DrainManagerTests.js diff --git a/services/real-time/test/unit/coffee/EventLoggerTests.coffee b/services/real-time/test/unit/coffee/EventLoggerTests.js similarity index 100% rename from services/real-time/test/unit/coffee/EventLoggerTests.coffee rename to services/real-time/test/unit/coffee/EventLoggerTests.js diff --git a/services/real-time/test/unit/coffee/RoomManagerTests.coffee b/services/real-time/test/unit/coffee/RoomManagerTests.js similarity index 100% rename from services/real-time/test/unit/coffee/RoomManagerTests.coffee rename to services/real-time/test/unit/coffee/RoomManagerTests.js diff --git a/services/real-time/test/unit/coffee/SafeJsonParseTest.coffee b/services/real-time/test/unit/coffee/SafeJsonParseTest.js similarity index 100% rename from services/real-time/test/unit/coffee/SafeJsonParseTest.coffee rename to services/real-time/test/unit/coffee/SafeJsonParseTest.js diff --git a/services/real-time/test/unit/coffee/SessionSocketsTests.coffee b/services/real-time/test/unit/coffee/SessionSocketsTests.js similarity index 100% rename from services/real-time/test/unit/coffee/SessionSocketsTests.coffee rename to services/real-time/test/unit/coffee/SessionSocketsTests.js diff --git a/services/real-time/test/unit/coffee/WebApiManagerTests.coffee b/services/real-time/test/unit/coffee/WebApiManagerTests.js similarity index 100% rename from services/real-time/test/unit/coffee/WebApiManagerTests.coffee rename to services/real-time/test/unit/coffee/WebApiManagerTests.js diff --git a/services/real-time/test/unit/coffee/WebsocketControllerTests.coffee b/services/real-time/test/unit/coffee/WebsocketControllerTests.js similarity index 100% rename from services/real-time/test/unit/coffee/WebsocketControllerTests.coffee rename to services/real-time/test/unit/coffee/WebsocketControllerTests.js diff --git a/services/real-time/test/unit/coffee/WebsocketLoadBalancerTests.coffee b/services/real-time/test/unit/coffee/WebsocketLoadBalancerTests.js similarity index 100% rename from services/real-time/test/unit/coffee/WebsocketLoadBalancerTests.coffee rename to services/real-time/test/unit/coffee/WebsocketLoadBalancerTests.js diff --git a/services/real-time/test/unit/coffee/helpers/MockClient.coffee b/services/real-time/test/unit/coffee/helpers/MockClient.js similarity index 100% rename from services/real-time/test/unit/coffee/helpers/MockClient.coffee rename to services/real-time/test/unit/coffee/helpers/MockClient.js From 2ca620e7a0184c0847c23c311282584781ff4369 Mon Sep 17 00:00:00 2001 From: decaffeinate Date: Tue, 23 Jun 2020 18:29:59 +0100 Subject: [PATCH 374/491] decaffeinate: Convert AuthorizationManagerTests.coffee and 13 other files to JS --- .../unit/coffee/AuthorizationManagerTests.js | 312 +-- .../test/unit/coffee/ChannelManagerTests.js | 402 ++-- .../unit/coffee/ConnectedUsersManagerTests.js | 325 +-- .../coffee/DocumentUpdaterControllerTests.js | 298 +-- .../coffee/DocumentUpdaterManagerTests.js | 447 +++-- .../test/unit/coffee/DrainManagerTests.js | 156 +- .../test/unit/coffee/EventLoggerTests.js | 151 +- .../test/unit/coffee/RoomManagerTests.js | 533 ++--- .../test/unit/coffee/SafeJsonParseTest.js | 75 +- .../test/unit/coffee/SessionSocketsTests.js | 250 ++- .../test/unit/coffee/WebApiManagerTests.js | 155 +- .../unit/coffee/WebsocketControllerTests.js | 1782 +++++++++-------- .../unit/coffee/WebsocketLoadBalancerTests.js | 307 +-- .../test/unit/coffee/helpers/MockClient.js | 25 +- 14 files changed, 2987 insertions(+), 2231 deletions(-) diff --git a/services/real-time/test/unit/coffee/AuthorizationManagerTests.js b/services/real-time/test/unit/coffee/AuthorizationManagerTests.js index 143218d8b2..626428ed61 100644 --- a/services/real-time/test/unit/coffee/AuthorizationManagerTests.js +++ b/services/real-time/test/unit/coffee/AuthorizationManagerTests.js @@ -1,166 +1,216 @@ -chai = require "chai" -chai.should() -expect = chai.expect -sinon = require("sinon") -SandboxedModule = require('sandboxed-module') -path = require "path" -modulePath = '../../../app/js/AuthorizationManager' +/* + * decaffeinate suggestions: + * DS102: Remove unnecessary code created because of implicit returns + * Full docs: https://github.com/decaffeinate/decaffeinate/blob/master/docs/suggestions.md + */ +const chai = require("chai"); +chai.should(); +const { + expect +} = chai; +const sinon = require("sinon"); +const SandboxedModule = require('sandboxed-module'); +const path = require("path"); +const modulePath = '../../../app/js/AuthorizationManager'; -describe 'AuthorizationManager', -> - beforeEach -> - @client = - ol_context: {} +describe('AuthorizationManager', function() { + beforeEach(function() { + this.client = + {ol_context: {}}; - @AuthorizationManager = SandboxedModule.require modulePath, requires: {} + return this.AuthorizationManager = SandboxedModule.require(modulePath, {requires: {}});}); - describe "assertClientCanViewProject", -> - it "should allow the readOnly privilegeLevel", (done) -> - @client.ol_context.privilege_level = "readOnly" - @AuthorizationManager.assertClientCanViewProject @client, (error) -> - expect(error).to.be.null - done() + describe("assertClientCanViewProject", function() { + it("should allow the readOnly privilegeLevel", function(done) { + this.client.ol_context.privilege_level = "readOnly"; + return this.AuthorizationManager.assertClientCanViewProject(this.client, function(error) { + expect(error).to.be.null; + return done(); + }); + }); - it "should allow the readAndWrite privilegeLevel", (done) -> - @client.ol_context.privilege_level = "readAndWrite" - @AuthorizationManager.assertClientCanViewProject @client, (error) -> - expect(error).to.be.null - done() + it("should allow the readAndWrite privilegeLevel", function(done) { + this.client.ol_context.privilege_level = "readAndWrite"; + return this.AuthorizationManager.assertClientCanViewProject(this.client, function(error) { + expect(error).to.be.null; + return done(); + }); + }); - it "should allow the owner privilegeLevel", (done) -> - @client.ol_context.privilege_level = "owner" - @AuthorizationManager.assertClientCanViewProject @client, (error) -> - expect(error).to.be.null - done() + it("should allow the owner privilegeLevel", function(done) { + this.client.ol_context.privilege_level = "owner"; + return this.AuthorizationManager.assertClientCanViewProject(this.client, function(error) { + expect(error).to.be.null; + return done(); + }); + }); - it "should return an error with any other privilegeLevel", (done) -> - @client.ol_context.privilege_level = "unknown" - @AuthorizationManager.assertClientCanViewProject @client, (error) -> - error.message.should.equal "not authorized" - done() + return it("should return an error with any other privilegeLevel", function(done) { + this.client.ol_context.privilege_level = "unknown"; + return this.AuthorizationManager.assertClientCanViewProject(this.client, function(error) { + error.message.should.equal("not authorized"); + return done(); + }); + }); + }); - describe "assertClientCanEditProject", -> - it "should not allow the readOnly privilegeLevel", (done) -> - @client.ol_context.privilege_level = "readOnly" - @AuthorizationManager.assertClientCanEditProject @client, (error) -> - error.message.should.equal "not authorized" - done() + describe("assertClientCanEditProject", function() { + it("should not allow the readOnly privilegeLevel", function(done) { + this.client.ol_context.privilege_level = "readOnly"; + return this.AuthorizationManager.assertClientCanEditProject(this.client, function(error) { + error.message.should.equal("not authorized"); + return done(); + }); + }); - it "should allow the readAndWrite privilegeLevel", (done) -> - @client.ol_context.privilege_level = "readAndWrite" - @AuthorizationManager.assertClientCanEditProject @client, (error) -> - expect(error).to.be.null - done() + it("should allow the readAndWrite privilegeLevel", function(done) { + this.client.ol_context.privilege_level = "readAndWrite"; + return this.AuthorizationManager.assertClientCanEditProject(this.client, function(error) { + expect(error).to.be.null; + return done(); + }); + }); - it "should allow the owner privilegeLevel", (done) -> - @client.ol_context.privilege_level = "owner" - @AuthorizationManager.assertClientCanEditProject @client, (error) -> - expect(error).to.be.null - done() + it("should allow the owner privilegeLevel", function(done) { + this.client.ol_context.privilege_level = "owner"; + return this.AuthorizationManager.assertClientCanEditProject(this.client, function(error) { + expect(error).to.be.null; + return done(); + }); + }); - it "should return an error with any other privilegeLevel", (done) -> - @client.ol_context.privilege_level = "unknown" - @AuthorizationManager.assertClientCanEditProject @client, (error) -> - error.message.should.equal "not authorized" - done() + return it("should return an error with any other privilegeLevel", function(done) { + this.client.ol_context.privilege_level = "unknown"; + return this.AuthorizationManager.assertClientCanEditProject(this.client, function(error) { + error.message.should.equal("not authorized"); + return done(); + }); + }); + }); - # check doc access for project + // check doc access for project - describe "assertClientCanViewProjectAndDoc", -> - beforeEach () -> - @doc_id = "12345" - @callback = sinon.stub() - @client.ol_context = {} + describe("assertClientCanViewProjectAndDoc", function() { + beforeEach(function() { + this.doc_id = "12345"; + this.callback = sinon.stub(); + return this.client.ol_context = {};}); - describe "when not authorised at the project level", -> - beforeEach () -> - @client.ol_context.privilege_level = "unknown" + describe("when not authorised at the project level", function() { + beforeEach(function() { + return this.client.ol_context.privilege_level = "unknown"; + }); - it "should not allow access", () -> - @AuthorizationManager.assertClientCanViewProjectAndDoc @client, @doc_id, (err) -> - err.message.should.equal "not authorized" + it("should not allow access", function() { + return this.AuthorizationManager.assertClientCanViewProjectAndDoc(this.client, this.doc_id, err => err.message.should.equal("not authorized")); + }); - describe "even when authorised at the doc level", -> - beforeEach (done) -> - @AuthorizationManager.addAccessToDoc @client, @doc_id, done + return describe("even when authorised at the doc level", function() { + beforeEach(function(done) { + return this.AuthorizationManager.addAccessToDoc(this.client, this.doc_id, done); + }); - it "should not allow access", () -> - @AuthorizationManager.assertClientCanViewProjectAndDoc @client, @doc_id, (err) -> - err.message.should.equal "not authorized" + return it("should not allow access", function() { + return this.AuthorizationManager.assertClientCanViewProjectAndDoc(this.client, this.doc_id, err => err.message.should.equal("not authorized")); + }); + }); + }); - describe "when authorised at the project level", -> - beforeEach () -> - @client.ol_context.privilege_level = "readOnly" + return describe("when authorised at the project level", function() { + beforeEach(function() { + return this.client.ol_context.privilege_level = "readOnly"; + }); - describe "and not authorised at the document level", -> - it "should not allow access", () -> - @AuthorizationManager.assertClientCanViewProjectAndDoc @client, @doc_id, (err) -> - err.message.should.equal "not authorized" + describe("and not authorised at the document level", () => it("should not allow access", function() { + return this.AuthorizationManager.assertClientCanViewProjectAndDoc(this.client, this.doc_id, err => err.message.should.equal("not authorized")); + })); - describe "and authorised at the document level", -> - beforeEach (done) -> - @AuthorizationManager.addAccessToDoc @client, @doc_id, done + describe("and authorised at the document level", function() { + beforeEach(function(done) { + return this.AuthorizationManager.addAccessToDoc(this.client, this.doc_id, done); + }); - it "should allow access", () -> - @AuthorizationManager.assertClientCanViewProjectAndDoc @client, @doc_id, @callback - @callback + return it("should allow access", function() { + this.AuthorizationManager.assertClientCanViewProjectAndDoc(this.client, this.doc_id, this.callback); + return this.callback .calledWith(null) - .should.equal true + .should.equal(true); + }); + }); - describe "when document authorisation is added and then removed", -> - beforeEach (done) -> - @AuthorizationManager.addAccessToDoc @client, @doc_id, () => - @AuthorizationManager.removeAccessToDoc @client, @doc_id, done + return describe("when document authorisation is added and then removed", function() { + beforeEach(function(done) { + return this.AuthorizationManager.addAccessToDoc(this.client, this.doc_id, () => { + return this.AuthorizationManager.removeAccessToDoc(this.client, this.doc_id, done); + }); + }); - it "should deny access", () -> - @AuthorizationManager.assertClientCanViewProjectAndDoc @client, @doc_id, (err) -> - err.message.should.equal "not authorized" + return it("should deny access", function() { + return this.AuthorizationManager.assertClientCanViewProjectAndDoc(this.client, this.doc_id, err => err.message.should.equal("not authorized")); + }); + }); + }); + }); - describe "assertClientCanEditProjectAndDoc", -> - beforeEach () -> - @doc_id = "12345" - @callback = sinon.stub() - @client.ol_context = {} + return describe("assertClientCanEditProjectAndDoc", function() { + beforeEach(function() { + this.doc_id = "12345"; + this.callback = sinon.stub(); + return this.client.ol_context = {};}); - describe "when not authorised at the project level", -> - beforeEach () -> - @client.ol_context.privilege_level = "readOnly" + describe("when not authorised at the project level", function() { + beforeEach(function() { + return this.client.ol_context.privilege_level = "readOnly"; + }); - it "should not allow access", () -> - @AuthorizationManager.assertClientCanEditProjectAndDoc @client, @doc_id, (err) -> - err.message.should.equal "not authorized" + it("should not allow access", function() { + return this.AuthorizationManager.assertClientCanEditProjectAndDoc(this.client, this.doc_id, err => err.message.should.equal("not authorized")); + }); - describe "even when authorised at the doc level", -> - beforeEach (done) -> - @AuthorizationManager.addAccessToDoc @client, @doc_id, done + return describe("even when authorised at the doc level", function() { + beforeEach(function(done) { + return this.AuthorizationManager.addAccessToDoc(this.client, this.doc_id, done); + }); - it "should not allow access", () -> - @AuthorizationManager.assertClientCanEditProjectAndDoc @client, @doc_id, (err) -> - err.message.should.equal "not authorized" + return it("should not allow access", function() { + return this.AuthorizationManager.assertClientCanEditProjectAndDoc(this.client, this.doc_id, err => err.message.should.equal("not authorized")); + }); + }); + }); - describe "when authorised at the project level", -> - beforeEach () -> - @client.ol_context.privilege_level = "readAndWrite" + return describe("when authorised at the project level", function() { + beforeEach(function() { + return this.client.ol_context.privilege_level = "readAndWrite"; + }); - describe "and not authorised at the document level", -> - it "should not allow access", () -> - @AuthorizationManager.assertClientCanEditProjectAndDoc @client, @doc_id, (err) -> - err.message.should.equal "not authorized" + describe("and not authorised at the document level", () => it("should not allow access", function() { + return this.AuthorizationManager.assertClientCanEditProjectAndDoc(this.client, this.doc_id, err => err.message.should.equal("not authorized")); + })); - describe "and authorised at the document level", -> - beforeEach (done) -> - @AuthorizationManager.addAccessToDoc @client, @doc_id, done + describe("and authorised at the document level", function() { + beforeEach(function(done) { + return this.AuthorizationManager.addAccessToDoc(this.client, this.doc_id, done); + }); - it "should allow access", () -> - @AuthorizationManager.assertClientCanEditProjectAndDoc @client, @doc_id, @callback - @callback + return it("should allow access", function() { + this.AuthorizationManager.assertClientCanEditProjectAndDoc(this.client, this.doc_id, this.callback); + return this.callback .calledWith(null) - .should.equal true + .should.equal(true); + }); + }); - describe "when document authorisation is added and then removed", -> - beforeEach (done) -> - @AuthorizationManager.addAccessToDoc @client, @doc_id, () => - @AuthorizationManager.removeAccessToDoc @client, @doc_id, done + return describe("when document authorisation is added and then removed", function() { + beforeEach(function(done) { + return this.AuthorizationManager.addAccessToDoc(this.client, this.doc_id, () => { + return this.AuthorizationManager.removeAccessToDoc(this.client, this.doc_id, done); + }); + }); - it "should deny access", () -> - @AuthorizationManager.assertClientCanEditProjectAndDoc @client, @doc_id, (err) -> - err.message.should.equal "not authorized" + return it("should deny access", function() { + return this.AuthorizationManager.assertClientCanEditProjectAndDoc(this.client, this.doc_id, err => err.message.should.equal("not authorized")); + }); + }); + }); + }); +}); diff --git a/services/real-time/test/unit/coffee/ChannelManagerTests.js b/services/real-time/test/unit/coffee/ChannelManagerTests.js index 354e956283..5c148451d0 100644 --- a/services/real-time/test/unit/coffee/ChannelManagerTests.js +++ b/services/real-time/test/unit/coffee/ChannelManagerTests.js @@ -1,220 +1,274 @@ -chai = require('chai') -should = chai.should() -expect = chai.expect -sinon = require("sinon") -modulePath = "../../../app/js/ChannelManager.js" -SandboxedModule = require('sandboxed-module') +/* + * decaffeinate suggestions: + * DS102: Remove unnecessary code created because of implicit returns + * Full docs: https://github.com/decaffeinate/decaffeinate/blob/master/docs/suggestions.md + */ +const chai = require('chai'); +const should = chai.should(); +const { + expect +} = chai; +const sinon = require("sinon"); +const modulePath = "../../../app/js/ChannelManager.js"; +const SandboxedModule = require('sandboxed-module'); -describe 'ChannelManager', -> - beforeEach -> - @rclient = {} - @other_rclient = {} - @ChannelManager = SandboxedModule.require modulePath, requires: - "settings-sharelatex": @settings = {} - "metrics-sharelatex": @metrics = {inc: sinon.stub(), summary: sinon.stub()} - "logger-sharelatex": @logger = { log: sinon.stub(), warn: sinon.stub(), error: sinon.stub() } +describe('ChannelManager', function() { + beforeEach(function() { + this.rclient = {}; + this.other_rclient = {}; + return this.ChannelManager = SandboxedModule.require(modulePath, { requires: { + "settings-sharelatex": (this.settings = {}), + "metrics-sharelatex": (this.metrics = {inc: sinon.stub(), summary: sinon.stub()}), + "logger-sharelatex": (this.logger = { log: sinon.stub(), warn: sinon.stub(), error: sinon.stub() }) + } + });}); - describe "subscribe", -> + describe("subscribe", function() { - describe "when there is no existing subscription for this redis client", -> - beforeEach (done) -> - @rclient.subscribe = sinon.stub().resolves() - @ChannelManager.subscribe @rclient, "applied-ops", "1234567890abcdef" - setTimeout done + describe("when there is no existing subscription for this redis client", function() { + beforeEach(function(done) { + this.rclient.subscribe = sinon.stub().resolves(); + this.ChannelManager.subscribe(this.rclient, "applied-ops", "1234567890abcdef"); + return setTimeout(done); + }); - it "should subscribe to the redis channel", -> - @rclient.subscribe.calledWithExactly("applied-ops:1234567890abcdef").should.equal true + return it("should subscribe to the redis channel", function() { + return this.rclient.subscribe.calledWithExactly("applied-ops:1234567890abcdef").should.equal(true); + }); + }); - describe "when there is an existing subscription for this redis client", -> - beforeEach (done) -> - @rclient.subscribe = sinon.stub().resolves() - @ChannelManager.subscribe @rclient, "applied-ops", "1234567890abcdef" - @ChannelManager.subscribe @rclient, "applied-ops", "1234567890abcdef" - setTimeout done + describe("when there is an existing subscription for this redis client", function() { + beforeEach(function(done) { + this.rclient.subscribe = sinon.stub().resolves(); + this.ChannelManager.subscribe(this.rclient, "applied-ops", "1234567890abcdef"); + this.ChannelManager.subscribe(this.rclient, "applied-ops", "1234567890abcdef"); + return setTimeout(done); + }); - it "should subscribe to the redis channel again", -> - @rclient.subscribe.callCount.should.equal 2 + return it("should subscribe to the redis channel again", function() { + return this.rclient.subscribe.callCount.should.equal(2); + }); + }); - describe "when subscribe errors", -> - beforeEach (done) -> - @rclient.subscribe = sinon.stub() + describe("when subscribe errors", function() { + beforeEach(function(done) { + this.rclient.subscribe = sinon.stub() .onFirstCall().rejects(new Error("some redis error")) - .onSecondCall().resolves() - p = @ChannelManager.subscribe @rclient, "applied-ops", "1234567890abcdef" - p.then () -> - done(new Error('should not subscribe but fail')) - .catch (err) => - err.message.should.equal "some redis error" - @ChannelManager.getClientMapEntry(@rclient).has("applied-ops:1234567890abcdef").should.equal false - @ChannelManager.subscribe @rclient, "applied-ops", "1234567890abcdef" - # subscribe is wrapped in Promise, delay other assertions - setTimeout done - return null + .onSecondCall().resolves(); + const p = this.ChannelManager.subscribe(this.rclient, "applied-ops", "1234567890abcdef"); + p.then(() => done(new Error('should not subscribe but fail'))).catch(err => { + err.message.should.equal("some redis error"); + this.ChannelManager.getClientMapEntry(this.rclient).has("applied-ops:1234567890abcdef").should.equal(false); + this.ChannelManager.subscribe(this.rclient, "applied-ops", "1234567890abcdef"); + // subscribe is wrapped in Promise, delay other assertions + return setTimeout(done); + }); + return null; + }); - it "should have recorded the error", -> - expect(@metrics.inc.calledWithExactly("subscribe.failed.applied-ops")).to.equal(true) + it("should have recorded the error", function() { + return expect(this.metrics.inc.calledWithExactly("subscribe.failed.applied-ops")).to.equal(true); + }); - it "should subscribe again", -> - @rclient.subscribe.callCount.should.equal 2 + it("should subscribe again", function() { + return this.rclient.subscribe.callCount.should.equal(2); + }); - it "should cleanup", -> - @ChannelManager.getClientMapEntry(@rclient).has("applied-ops:1234567890abcdef").should.equal false + return it("should cleanup", function() { + return this.ChannelManager.getClientMapEntry(this.rclient).has("applied-ops:1234567890abcdef").should.equal(false); + }); + }); - describe "when subscribe errors and the clientChannelMap entry was replaced", -> - beforeEach (done) -> - @rclient.subscribe = sinon.stub() + describe("when subscribe errors and the clientChannelMap entry was replaced", function() { + beforeEach(function(done) { + this.rclient.subscribe = sinon.stub() .onFirstCall().rejects(new Error("some redis error")) - .onSecondCall().resolves() - @first = @ChannelManager.subscribe @rclient, "applied-ops", "1234567890abcdef" - # ignore error - @first.catch((()->)) - expect(@ChannelManager.getClientMapEntry(@rclient).get("applied-ops:1234567890abcdef")).to.equal @first + .onSecondCall().resolves(); + this.first = this.ChannelManager.subscribe(this.rclient, "applied-ops", "1234567890abcdef"); + // ignore error + this.first.catch((function(){})); + expect(this.ChannelManager.getClientMapEntry(this.rclient).get("applied-ops:1234567890abcdef")).to.equal(this.first); - @rclient.unsubscribe = sinon.stub().resolves() - @ChannelManager.unsubscribe @rclient, "applied-ops", "1234567890abcdef" - @second = @ChannelManager.subscribe @rclient, "applied-ops", "1234567890abcdef" - # should get replaced immediately - expect(@ChannelManager.getClientMapEntry(@rclient).get("applied-ops:1234567890abcdef")).to.equal @second + this.rclient.unsubscribe = sinon.stub().resolves(); + this.ChannelManager.unsubscribe(this.rclient, "applied-ops", "1234567890abcdef"); + this.second = this.ChannelManager.subscribe(this.rclient, "applied-ops", "1234567890abcdef"); + // should get replaced immediately + expect(this.ChannelManager.getClientMapEntry(this.rclient).get("applied-ops:1234567890abcdef")).to.equal(this.second); - # let the first subscribe error -> unsubscribe -> subscribe - setTimeout done + // let the first subscribe error -> unsubscribe -> subscribe + return setTimeout(done); + }); - it "should cleanup the second subscribePromise", -> - expect(@ChannelManager.getClientMapEntry(@rclient).has("applied-ops:1234567890abcdef")).to.equal false + return it("should cleanup the second subscribePromise", function() { + return expect(this.ChannelManager.getClientMapEntry(this.rclient).has("applied-ops:1234567890abcdef")).to.equal(false); + }); + }); - describe "when there is an existing subscription for another redis client but not this one", -> - beforeEach (done) -> - @other_rclient.subscribe = sinon.stub().resolves() - @ChannelManager.subscribe @other_rclient, "applied-ops", "1234567890abcdef" - @rclient.subscribe = sinon.stub().resolves() # discard the original stub - @ChannelManager.subscribe @rclient, "applied-ops", "1234567890abcdef" - setTimeout done + return describe("when there is an existing subscription for another redis client but not this one", function() { + beforeEach(function(done) { + this.other_rclient.subscribe = sinon.stub().resolves(); + this.ChannelManager.subscribe(this.other_rclient, "applied-ops", "1234567890abcdef"); + this.rclient.subscribe = sinon.stub().resolves(); // discard the original stub + this.ChannelManager.subscribe(this.rclient, "applied-ops", "1234567890abcdef"); + return setTimeout(done); + }); - it "should subscribe to the redis channel on this redis client", -> - @rclient.subscribe.calledWithExactly("applied-ops:1234567890abcdef").should.equal true + return it("should subscribe to the redis channel on this redis client", function() { + return this.rclient.subscribe.calledWithExactly("applied-ops:1234567890abcdef").should.equal(true); + }); + }); + }); - describe "unsubscribe", -> + describe("unsubscribe", function() { - describe "when there is no existing subscription for this redis client", -> - beforeEach (done) -> - @rclient.unsubscribe = sinon.stub().resolves() - @ChannelManager.unsubscribe @rclient, "applied-ops", "1234567890abcdef" - setTimeout done + describe("when there is no existing subscription for this redis client", function() { + beforeEach(function(done) { + this.rclient.unsubscribe = sinon.stub().resolves(); + this.ChannelManager.unsubscribe(this.rclient, "applied-ops", "1234567890abcdef"); + return setTimeout(done); + }); - it "should unsubscribe from the redis channel", -> - @rclient.unsubscribe.called.should.equal true + return it("should unsubscribe from the redis channel", function() { + return this.rclient.unsubscribe.called.should.equal(true); + }); + }); - describe "when there is an existing subscription for this another redis client but not this one", -> - beforeEach (done) -> - @other_rclient.subscribe = sinon.stub().resolves() - @rclient.unsubscribe = sinon.stub().resolves() - @ChannelManager.subscribe @other_rclient, "applied-ops", "1234567890abcdef" - @ChannelManager.unsubscribe @rclient, "applied-ops", "1234567890abcdef" - setTimeout done + describe("when there is an existing subscription for this another redis client but not this one", function() { + beforeEach(function(done) { + this.other_rclient.subscribe = sinon.stub().resolves(); + this.rclient.unsubscribe = sinon.stub().resolves(); + this.ChannelManager.subscribe(this.other_rclient, "applied-ops", "1234567890abcdef"); + this.ChannelManager.unsubscribe(this.rclient, "applied-ops", "1234567890abcdef"); + return setTimeout(done); + }); - it "should still unsubscribe from the redis channel on this client", -> - @rclient.unsubscribe.called.should.equal true + return it("should still unsubscribe from the redis channel on this client", function() { + return this.rclient.unsubscribe.called.should.equal(true); + }); + }); - describe "when unsubscribe errors and completes", -> - beforeEach (done) -> - @rclient.subscribe = sinon.stub().resolves() - @ChannelManager.subscribe @rclient, "applied-ops", "1234567890abcdef" - @rclient.unsubscribe = sinon.stub().rejects(new Error("some redis error")) - @ChannelManager.unsubscribe @rclient, "applied-ops", "1234567890abcdef" - setTimeout done - return null + describe("when unsubscribe errors and completes", function() { + beforeEach(function(done) { + this.rclient.subscribe = sinon.stub().resolves(); + this.ChannelManager.subscribe(this.rclient, "applied-ops", "1234567890abcdef"); + this.rclient.unsubscribe = sinon.stub().rejects(new Error("some redis error")); + this.ChannelManager.unsubscribe(this.rclient, "applied-ops", "1234567890abcdef"); + setTimeout(done); + return null; + }); - it "should have cleaned up", -> - @ChannelManager.getClientMapEntry(@rclient).has("applied-ops:1234567890abcdef").should.equal false + it("should have cleaned up", function() { + return this.ChannelManager.getClientMapEntry(this.rclient).has("applied-ops:1234567890abcdef").should.equal(false); + }); - it "should not error out when subscribing again", (done) -> - p = @ChannelManager.subscribe @rclient, "applied-ops", "1234567890abcdef" - p.then () -> - done() - .catch done - return null + return it("should not error out when subscribing again", function(done) { + const p = this.ChannelManager.subscribe(this.rclient, "applied-ops", "1234567890abcdef"); + p.then(() => done()).catch(done); + return null; + }); + }); - describe "when unsubscribe errors and another client subscribes at the same time", -> - beforeEach (done) -> - @rclient.subscribe = sinon.stub().resolves() - @ChannelManager.subscribe @rclient, "applied-ops", "1234567890abcdef" - rejectSubscribe = undefined - @rclient.unsubscribe = () -> - return new Promise (resolve, reject) -> - rejectSubscribe = reject - @ChannelManager.unsubscribe @rclient, "applied-ops", "1234567890abcdef" + describe("when unsubscribe errors and another client subscribes at the same time", function() { + beforeEach(function(done) { + this.rclient.subscribe = sinon.stub().resolves(); + this.ChannelManager.subscribe(this.rclient, "applied-ops", "1234567890abcdef"); + let rejectSubscribe = undefined; + this.rclient.unsubscribe = () => new Promise((resolve, reject) => rejectSubscribe = reject); + this.ChannelManager.unsubscribe(this.rclient, "applied-ops", "1234567890abcdef"); - setTimeout () => - # delay, actualUnsubscribe should not see the new subscribe request - @ChannelManager.subscribe(@rclient, "applied-ops", "1234567890abcdef") - .then () -> - setTimeout done - .catch done - setTimeout -> - # delay, rejectSubscribe is not defined immediately - rejectSubscribe(new Error("redis error")) - return null + setTimeout(() => { + // delay, actualUnsubscribe should not see the new subscribe request + this.ChannelManager.subscribe(this.rclient, "applied-ops", "1234567890abcdef") + .then(() => setTimeout(done)).catch(done); + return setTimeout(() => // delay, rejectSubscribe is not defined immediately + rejectSubscribe(new Error("redis error"))); + }); + return null; + }); - it "should have recorded the error", -> - expect(@metrics.inc.calledWithExactly("unsubscribe.failed.applied-ops")).to.equal(true) + it("should have recorded the error", function() { + return expect(this.metrics.inc.calledWithExactly("unsubscribe.failed.applied-ops")).to.equal(true); + }); - it "should have subscribed", -> - @rclient.subscribe.called.should.equal true + it("should have subscribed", function() { + return this.rclient.subscribe.called.should.equal(true); + }); - it "should have discarded the finished Promise", -> - @ChannelManager.getClientMapEntry(@rclient).has("applied-ops:1234567890abcdef").should.equal false + return it("should have discarded the finished Promise", function() { + return this.ChannelManager.getClientMapEntry(this.rclient).has("applied-ops:1234567890abcdef").should.equal(false); + }); + }); - describe "when there is an existing subscription for this redis client", -> - beforeEach (done) -> - @rclient.subscribe = sinon.stub().resolves() - @rclient.unsubscribe = sinon.stub().resolves() - @ChannelManager.subscribe @rclient, "applied-ops", "1234567890abcdef" - @ChannelManager.unsubscribe @rclient, "applied-ops", "1234567890abcdef" - setTimeout done + return describe("when there is an existing subscription for this redis client", function() { + beforeEach(function(done) { + this.rclient.subscribe = sinon.stub().resolves(); + this.rclient.unsubscribe = sinon.stub().resolves(); + this.ChannelManager.subscribe(this.rclient, "applied-ops", "1234567890abcdef"); + this.ChannelManager.unsubscribe(this.rclient, "applied-ops", "1234567890abcdef"); + return setTimeout(done); + }); - it "should unsubscribe from the redis channel", -> - @rclient.unsubscribe.calledWithExactly("applied-ops:1234567890abcdef").should.equal true + return it("should unsubscribe from the redis channel", function() { + return this.rclient.unsubscribe.calledWithExactly("applied-ops:1234567890abcdef").should.equal(true); + }); + }); + }); - describe "publish", -> + return describe("publish", function() { - describe "when the channel is 'all'", -> - beforeEach -> - @rclient.publish = sinon.stub() - @ChannelManager.publish @rclient, "applied-ops", "all", "random-message" + describe("when the channel is 'all'", function() { + beforeEach(function() { + this.rclient.publish = sinon.stub(); + return this.ChannelManager.publish(this.rclient, "applied-ops", "all", "random-message"); + }); - it "should publish on the base channel", -> - @rclient.publish.calledWithExactly("applied-ops", "random-message").should.equal true + return it("should publish on the base channel", function() { + return this.rclient.publish.calledWithExactly("applied-ops", "random-message").should.equal(true); + }); + }); - describe "when the channel has an specific id", -> + describe("when the channel has an specific id", function() { - describe "when the individual channel setting is false", -> - beforeEach -> - @rclient.publish = sinon.stub() - @settings.publishOnIndividualChannels = false - @ChannelManager.publish @rclient, "applied-ops", "1234567890abcdef", "random-message" + describe("when the individual channel setting is false", function() { + beforeEach(function() { + this.rclient.publish = sinon.stub(); + this.settings.publishOnIndividualChannels = false; + return this.ChannelManager.publish(this.rclient, "applied-ops", "1234567890abcdef", "random-message"); + }); - it "should publish on the per-id channel", -> - @rclient.publish.calledWithExactly("applied-ops", "random-message").should.equal true - @rclient.publish.calledOnce.should.equal true + return it("should publish on the per-id channel", function() { + this.rclient.publish.calledWithExactly("applied-ops", "random-message").should.equal(true); + return this.rclient.publish.calledOnce.should.equal(true); + }); + }); - describe "when the individual channel setting is true", -> - beforeEach -> - @rclient.publish = sinon.stub() - @settings.publishOnIndividualChannels = true - @ChannelManager.publish @rclient, "applied-ops", "1234567890abcdef", "random-message" + return describe("when the individual channel setting is true", function() { + beforeEach(function() { + this.rclient.publish = sinon.stub(); + this.settings.publishOnIndividualChannels = true; + return this.ChannelManager.publish(this.rclient, "applied-ops", "1234567890abcdef", "random-message"); + }); - it "should publish on the per-id channel", -> - @rclient.publish.calledWithExactly("applied-ops:1234567890abcdef", "random-message").should.equal true - @rclient.publish.calledOnce.should.equal true + return it("should publish on the per-id channel", function() { + this.rclient.publish.calledWithExactly("applied-ops:1234567890abcdef", "random-message").should.equal(true); + return this.rclient.publish.calledOnce.should.equal(true); + }); + }); + }); - describe "metrics", -> - beforeEach -> - @rclient.publish = sinon.stub() - @ChannelManager.publish @rclient, "applied-ops", "all", "random-message" + return describe("metrics", function() { + beforeEach(function() { + this.rclient.publish = sinon.stub(); + return this.ChannelManager.publish(this.rclient, "applied-ops", "all", "random-message"); + }); - it "should track the payload size", -> - @metrics.summary.calledWithExactly( + return it("should track the payload size", function() { + return this.metrics.summary.calledWithExactly( "redis.publish.applied-ops", "random-message".length - ).should.equal true + ).should.equal(true); + }); + }); + }); +}); diff --git a/services/real-time/test/unit/coffee/ConnectedUsersManagerTests.js b/services/real-time/test/unit/coffee/ConnectedUsersManagerTests.js index 6fb3942b64..c1657e3669 100644 --- a/services/real-time/test/unit/coffee/ConnectedUsersManagerTests.js +++ b/services/real-time/test/unit/coffee/ConnectedUsersManagerTests.js @@ -1,164 +1,221 @@ +/* + * decaffeinate suggestions: + * DS102: Remove unnecessary code created because of implicit returns + * Full docs: https://github.com/decaffeinate/decaffeinate/blob/master/docs/suggestions.md + */ -should = require('chai').should() -SandboxedModule = require('sandboxed-module') -assert = require('assert') -path = require('path') -sinon = require('sinon') -modulePath = path.join __dirname, "../../../app/js/ConnectedUsersManager" -expect = require("chai").expect -tk = require("timekeeper") +const should = require('chai').should(); +const SandboxedModule = require('sandboxed-module'); +const assert = require('assert'); +const path = require('path'); +const sinon = require('sinon'); +const modulePath = path.join(__dirname, "../../../app/js/ConnectedUsersManager"); +const { + expect +} = require("chai"); +const tk = require("timekeeper"); -describe "ConnectedUsersManager", -> +describe("ConnectedUsersManager", function() { - beforeEach -> + beforeEach(function() { - @settings = - redis: - realtime: - key_schema: - clientsInProject: ({project_id}) -> "clients_in_project:#{project_id}" - connectedUser: ({project_id, client_id})-> "connected_user:#{project_id}:#{client_id}" - @rClient = - auth:-> - setex:sinon.stub() - sadd:sinon.stub() - get: sinon.stub() - srem:sinon.stub() - del:sinon.stub() - smembers:sinon.stub() - expire:sinon.stub() - hset:sinon.stub() - hgetall:sinon.stub() - exec:sinon.stub() - multi: => return @rClient - tk.freeze(new Date()) + this.settings = { + redis: { + realtime: { + key_schema: { + clientsInProject({project_id}) { return `clients_in_project:${project_id}`; }, + connectedUser({project_id, client_id}){ return `connected_user:${project_id}:${client_id}`; } + } + } + } + }; + this.rClient = { + auth() {}, + setex:sinon.stub(), + sadd:sinon.stub(), + get: sinon.stub(), + srem:sinon.stub(), + del:sinon.stub(), + smembers:sinon.stub(), + expire:sinon.stub(), + hset:sinon.stub(), + hgetall:sinon.stub(), + exec:sinon.stub(), + multi: () => { return this.rClient; } + }; + tk.freeze(new Date()); - @ConnectedUsersManager = SandboxedModule.require modulePath, requires: - "settings-sharelatex":@settings - "logger-sharelatex": log:-> - "redis-sharelatex": createClient:=> - return @rClient - @client_id = "32132132" - @project_id = "dskjh2u21321" - @user = { - _id: "user-id-123" - first_name: "Joe" - last_name: "Bloggs" - email: "joe@example.com" + this.ConnectedUsersManager = SandboxedModule.require(modulePath, { requires: { + "settings-sharelatex":this.settings, + "logger-sharelatex": { log() {} + }, + "redis-sharelatex": { createClient:() => { + return this.rClient; + } } - @cursorData = { row: 12, column: 9, doc_id: '53c3b8c85fee64000023dc6e' } + } + } + ); + this.client_id = "32132132"; + this.project_id = "dskjh2u21321"; + this.user = { + _id: "user-id-123", + first_name: "Joe", + last_name: "Bloggs", + email: "joe@example.com" + }; + return this.cursorData = { row: 12, column: 9, doc_id: '53c3b8c85fee64000023dc6e' };}); - afterEach -> - tk.reset() + afterEach(() => tk.reset()); - describe "updateUserPosition", -> - beforeEach -> - @rClient.exec.callsArgWith(0) + describe("updateUserPosition", function() { + beforeEach(function() { + return this.rClient.exec.callsArgWith(0); + }); - it "should set a key with the date and give it a ttl", (done)-> - @ConnectedUsersManager.updateUserPosition @project_id, @client_id, @user, null, (err)=> - @rClient.hset.calledWith("connected_user:#{@project_id}:#{@client_id}", "last_updated_at", Date.now()).should.equal true - done() + it("should set a key with the date and give it a ttl", function(done){ + return this.ConnectedUsersManager.updateUserPosition(this.project_id, this.client_id, this.user, null, err=> { + this.rClient.hset.calledWith(`connected_user:${this.project_id}:${this.client_id}`, "last_updated_at", Date.now()).should.equal(true); + return done(); + }); + }); - it "should set a key with the user_id", (done)-> - @ConnectedUsersManager.updateUserPosition @project_id, @client_id, @user, null, (err)=> - @rClient.hset.calledWith("connected_user:#{@project_id}:#{@client_id}", "user_id", @user._id).should.equal true - done() + it("should set a key with the user_id", function(done){ + return this.ConnectedUsersManager.updateUserPosition(this.project_id, this.client_id, this.user, null, err=> { + this.rClient.hset.calledWith(`connected_user:${this.project_id}:${this.client_id}`, "user_id", this.user._id).should.equal(true); + return done(); + }); + }); - it "should set a key with the first_name", (done)-> - @ConnectedUsersManager.updateUserPosition @project_id, @client_id, @user, null, (err)=> - @rClient.hset.calledWith("connected_user:#{@project_id}:#{@client_id}", "first_name", @user.first_name).should.equal true - done() + it("should set a key with the first_name", function(done){ + return this.ConnectedUsersManager.updateUserPosition(this.project_id, this.client_id, this.user, null, err=> { + this.rClient.hset.calledWith(`connected_user:${this.project_id}:${this.client_id}`, "first_name", this.user.first_name).should.equal(true); + return done(); + }); + }); - it "should set a key with the last_name", (done)-> - @ConnectedUsersManager.updateUserPosition @project_id, @client_id, @user, null, (err)=> - @rClient.hset.calledWith("connected_user:#{@project_id}:#{@client_id}", "last_name", @user.last_name).should.equal true - done() + it("should set a key with the last_name", function(done){ + return this.ConnectedUsersManager.updateUserPosition(this.project_id, this.client_id, this.user, null, err=> { + this.rClient.hset.calledWith(`connected_user:${this.project_id}:${this.client_id}`, "last_name", this.user.last_name).should.equal(true); + return done(); + }); + }); - it "should set a key with the email", (done)-> - @ConnectedUsersManager.updateUserPosition @project_id, @client_id, @user, null, (err)=> - @rClient.hset.calledWith("connected_user:#{@project_id}:#{@client_id}", "email", @user.email).should.equal true - done() + it("should set a key with the email", function(done){ + return this.ConnectedUsersManager.updateUserPosition(this.project_id, this.client_id, this.user, null, err=> { + this.rClient.hset.calledWith(`connected_user:${this.project_id}:${this.client_id}`, "email", this.user.email).should.equal(true); + return done(); + }); + }); - it "should push the client_id on to the project list", (done)-> - @ConnectedUsersManager.updateUserPosition @project_id, @client_id, @user, null, (err)=> - @rClient.sadd.calledWith("clients_in_project:#{@project_id}", @client_id).should.equal true - done() + it("should push the client_id on to the project list", function(done){ + return this.ConnectedUsersManager.updateUserPosition(this.project_id, this.client_id, this.user, null, err=> { + this.rClient.sadd.calledWith(`clients_in_project:${this.project_id}`, this.client_id).should.equal(true); + return done(); + }); + }); - it "should add a ttl to the project set so it stays clean", (done)-> - @ConnectedUsersManager.updateUserPosition @project_id, @client_id, @user, null, (err)=> - @rClient.expire.calledWith("clients_in_project:#{@project_id}", 24 * 4 * 60 * 60).should.equal true - done() + it("should add a ttl to the project set so it stays clean", function(done){ + return this.ConnectedUsersManager.updateUserPosition(this.project_id, this.client_id, this.user, null, err=> { + this.rClient.expire.calledWith(`clients_in_project:${this.project_id}`, 24 * 4 * 60 * 60).should.equal(true); + return done(); + }); + }); - it "should add a ttl to the connected user so it stays clean", (done) -> - @ConnectedUsersManager.updateUserPosition @project_id, @client_id, @user, null, (err)=> - @rClient.expire.calledWith("connected_user:#{@project_id}:#{@client_id}", 60 * 15).should.equal true - done() + it("should add a ttl to the connected user so it stays clean", function(done) { + return this.ConnectedUsersManager.updateUserPosition(this.project_id, this.client_id, this.user, null, err=> { + this.rClient.expire.calledWith(`connected_user:${this.project_id}:${this.client_id}`, 60 * 15).should.equal(true); + return done(); + }); + }); - it "should set the cursor position when provided", (done)-> - @ConnectedUsersManager.updateUserPosition @project_id, @client_id, @user, @cursorData, (err)=> - @rClient.hset.calledWith("connected_user:#{@project_id}:#{@client_id}", "cursorData", JSON.stringify(@cursorData)).should.equal true - done() + return it("should set the cursor position when provided", function(done){ + return this.ConnectedUsersManager.updateUserPosition(this.project_id, this.client_id, this.user, this.cursorData, err=> { + this.rClient.hset.calledWith(`connected_user:${this.project_id}:${this.client_id}`, "cursorData", JSON.stringify(this.cursorData)).should.equal(true); + return done(); + }); + }); + }); - describe "markUserAsDisconnected", -> - beforeEach -> - @rClient.exec.callsArgWith(0) + describe("markUserAsDisconnected", function() { + beforeEach(function() { + return this.rClient.exec.callsArgWith(0); + }); - it "should remove the user from the set", (done)-> - @ConnectedUsersManager.markUserAsDisconnected @project_id, @client_id, (err)=> - @rClient.srem.calledWith("clients_in_project:#{@project_id}", @client_id).should.equal true - done() + it("should remove the user from the set", function(done){ + return this.ConnectedUsersManager.markUserAsDisconnected(this.project_id, this.client_id, err=> { + this.rClient.srem.calledWith(`clients_in_project:${this.project_id}`, this.client_id).should.equal(true); + return done(); + }); + }); - it "should delete the connected_user string", (done)-> - @ConnectedUsersManager.markUserAsDisconnected @project_id, @client_id, (err)=> - @rClient.del.calledWith("connected_user:#{@project_id}:#{@client_id}").should.equal true - done() + it("should delete the connected_user string", function(done){ + return this.ConnectedUsersManager.markUserAsDisconnected(this.project_id, this.client_id, err=> { + this.rClient.del.calledWith(`connected_user:${this.project_id}:${this.client_id}`).should.equal(true); + return done(); + }); + }); - it "should add a ttl to the connected user set so it stays clean", (done)-> - @ConnectedUsersManager.markUserAsDisconnected @project_id, @client_id, (err)=> - @rClient.expire.calledWith("clients_in_project:#{@project_id}", 24 * 4 * 60 * 60).should.equal true - done() + return it("should add a ttl to the connected user set so it stays clean", function(done){ + return this.ConnectedUsersManager.markUserAsDisconnected(this.project_id, this.client_id, err=> { + this.rClient.expire.calledWith(`clients_in_project:${this.project_id}`, 24 * 4 * 60 * 60).should.equal(true); + return done(); + }); + }); + }); - describe "_getConnectedUser", -> + describe("_getConnectedUser", function() { - it "should return a connected user if there is a user object", (done)-> - cursorData = JSON.stringify(cursorData:{row:1}) - @rClient.hgetall.callsArgWith(1, null, {connected_at:new Date(), user_id: @user._id, last_updated_at: "#{Date.now()}", cursorData}) - @ConnectedUsersManager._getConnectedUser @project_id, @client_id, (err, result)=> - result.connected.should.equal true - result.client_id.should.equal @client_id - done() + it("should return a connected user if there is a user object", function(done){ + const cursorData = JSON.stringify({cursorData:{row:1}}); + this.rClient.hgetall.callsArgWith(1, null, {connected_at:new Date(), user_id: this.user._id, last_updated_at: `${Date.now()}`, cursorData}); + return this.ConnectedUsersManager._getConnectedUser(this.project_id, this.client_id, (err, result)=> { + result.connected.should.equal(true); + result.client_id.should.equal(this.client_id); + return done(); + }); + }); - it "should return a not connected user if there is no object", (done)-> - @rClient.hgetall.callsArgWith(1, null, null) - @ConnectedUsersManager._getConnectedUser @project_id, @client_id, (err, result)=> - result.connected.should.equal false - result.client_id.should.equal @client_id - done() + it("should return a not connected user if there is no object", function(done){ + this.rClient.hgetall.callsArgWith(1, null, null); + return this.ConnectedUsersManager._getConnectedUser(this.project_id, this.client_id, (err, result)=> { + result.connected.should.equal(false); + result.client_id.should.equal(this.client_id); + return done(); + }); + }); - it "should return a not connected user if there is an empty object", (done)-> - @rClient.hgetall.callsArgWith(1, null, {}) - @ConnectedUsersManager._getConnectedUser @project_id, @client_id, (err, result)=> - result.connected.should.equal false - result.client_id.should.equal @client_id - done() + return it("should return a not connected user if there is an empty object", function(done){ + this.rClient.hgetall.callsArgWith(1, null, {}); + return this.ConnectedUsersManager._getConnectedUser(this.project_id, this.client_id, (err, result)=> { + result.connected.should.equal(false); + result.client_id.should.equal(this.client_id); + return done(); + }); + }); + }); - describe "getConnectedUsers", -> + return describe("getConnectedUsers", function() { - beforeEach -> - @users = ["1234", "5678", "9123", "8234"] - @rClient.smembers.callsArgWith(1, null, @users) - @ConnectedUsersManager._getConnectedUser = sinon.stub() - @ConnectedUsersManager._getConnectedUser.withArgs(@project_id, @users[0]).callsArgWith(2, null, {connected:true, client_age: 2, client_id:@users[0]}) - @ConnectedUsersManager._getConnectedUser.withArgs(@project_id, @users[1]).callsArgWith(2, null, {connected:false, client_age: 1, client_id:@users[1]}) - @ConnectedUsersManager._getConnectedUser.withArgs(@project_id, @users[2]).callsArgWith(2, null, {connected:true, client_age: 3, client_id:@users[2]}) - @ConnectedUsersManager._getConnectedUser.withArgs(@project_id, @users[3]).callsArgWith(2, null, {connected:true, client_age: 11, client_id:@users[3]}) # connected but old + beforeEach(function() { + this.users = ["1234", "5678", "9123", "8234"]; + this.rClient.smembers.callsArgWith(1, null, this.users); + this.ConnectedUsersManager._getConnectedUser = sinon.stub(); + this.ConnectedUsersManager._getConnectedUser.withArgs(this.project_id, this.users[0]).callsArgWith(2, null, {connected:true, client_age: 2, client_id:this.users[0]}); + this.ConnectedUsersManager._getConnectedUser.withArgs(this.project_id, this.users[1]).callsArgWith(2, null, {connected:false, client_age: 1, client_id:this.users[1]}); + this.ConnectedUsersManager._getConnectedUser.withArgs(this.project_id, this.users[2]).callsArgWith(2, null, {connected:true, client_age: 3, client_id:this.users[2]}); + return this.ConnectedUsersManager._getConnectedUser.withArgs(this.project_id, this.users[3]).callsArgWith(2, null, {connected:true, client_age: 11, client_id:this.users[3]}); + }); // connected but old - it "should only return the users in the list which are still in redis and recently updated", (done)-> - @ConnectedUsersManager.getConnectedUsers @project_id, (err, users)=> - users.length.should.equal 2 - users[0].should.deep.equal {client_id:@users[0], client_age: 2, connected:true} - users[1].should.deep.equal {client_id:@users[2], client_age: 3, connected:true} - done() + return it("should only return the users in the list which are still in redis and recently updated", function(done){ + return this.ConnectedUsersManager.getConnectedUsers(this.project_id, (err, users)=> { + users.length.should.equal(2); + users[0].should.deep.equal({client_id:this.users[0], client_age: 2, connected:true}); + users[1].should.deep.equal({client_id:this.users[2], client_age: 3, connected:true}); + return done(); + }); + }); + }); +}); diff --git a/services/real-time/test/unit/coffee/DocumentUpdaterControllerTests.js b/services/real-time/test/unit/coffee/DocumentUpdaterControllerTests.js index b2e52c7d56..9d15a77394 100644 --- a/services/real-time/test/unit/coffee/DocumentUpdaterControllerTests.js +++ b/services/real-time/test/unit/coffee/DocumentUpdaterControllerTests.js @@ -1,153 +1,203 @@ -SandboxedModule = require('sandboxed-module') -sinon = require('sinon') -require('chai').should() -modulePath = require('path').join __dirname, '../../../app/js/DocumentUpdaterController' -MockClient = require "./helpers/MockClient" +/* + * decaffeinate suggestions: + * DS101: Remove unnecessary use of Array.from + * DS102: Remove unnecessary code created because of implicit returns + * Full docs: https://github.com/decaffeinate/decaffeinate/blob/master/docs/suggestions.md + */ +const SandboxedModule = require('sandboxed-module'); +const sinon = require('sinon'); +require('chai').should(); +const modulePath = require('path').join(__dirname, '../../../app/js/DocumentUpdaterController'); +const MockClient = require("./helpers/MockClient"); -describe "DocumentUpdaterController", -> - beforeEach -> - @project_id = "project-id-123" - @doc_id = "doc-id-123" - @callback = sinon.stub() - @io = { "mock": "socket.io" } - @rclient = [] - @RoomEvents = { on: sinon.stub() } - @EditorUpdatesController = SandboxedModule.require modulePath, requires: - "logger-sharelatex": @logger = { error: sinon.stub(), log: sinon.stub(), warn: sinon.stub() } - "settings-sharelatex": @settings = - redis: - documentupdater: - key_schema: - pendingUpdates: ({doc_id}) -> "PendingUpdates:#{doc_id}" +describe("DocumentUpdaterController", function() { + beforeEach(function() { + this.project_id = "project-id-123"; + this.doc_id = "doc-id-123"; + this.callback = sinon.stub(); + this.io = { "mock": "socket.io" }; + this.rclient = []; + this.RoomEvents = { on: sinon.stub() }; + return this.EditorUpdatesController = SandboxedModule.require(modulePath, { requires: { + "logger-sharelatex": (this.logger = { error: sinon.stub(), log: sinon.stub(), warn: sinon.stub() }), + "settings-sharelatex": (this.settings = { + redis: { + documentupdater: { + key_schema: { + pendingUpdates({doc_id}) { return `PendingUpdates:${doc_id}`; } + } + }, pubsub: null - "redis-sharelatex" : @redis = - createClient: (name) => - @rclient.push(rclientStub = {name:name}) - return rclientStub - "./SafeJsonParse": @SafeJsonParse = - parse: (data, cb) => cb null, JSON.parse(data) - "./EventLogger": @EventLogger = {checkEventOrder: sinon.stub()} - "./HealthCheckManager": {check: sinon.stub()} - "metrics-sharelatex": @metrics = {inc: sinon.stub()} - "./RoomManager" : @RoomManager = { eventSource: sinon.stub().returns @RoomEvents} - "./ChannelManager": @ChannelManager = {} + } + }), + "redis-sharelatex" : (this.redis = { + createClient: name => { + let rclientStub; + this.rclient.push(rclientStub = {name}); + return rclientStub; + } + }), + "./SafeJsonParse": (this.SafeJsonParse = + {parse: (data, cb) => cb(null, JSON.parse(data))}), + "./EventLogger": (this.EventLogger = {checkEventOrder: sinon.stub()}), + "./HealthCheckManager": {check: sinon.stub()}, + "metrics-sharelatex": (this.metrics = {inc: sinon.stub()}), + "./RoomManager" : (this.RoomManager = { eventSource: sinon.stub().returns(this.RoomEvents)}), + "./ChannelManager": (this.ChannelManager = {}) + } + });}); - describe "listenForUpdatesFromDocumentUpdater", -> - beforeEach -> - @rclient.length = 0 # clear any existing clients - @EditorUpdatesController.rclientList = [@redis.createClient("first"), @redis.createClient("second")] - @rclient[0].subscribe = sinon.stub() - @rclient[0].on = sinon.stub() - @rclient[1].subscribe = sinon.stub() - @rclient[1].on = sinon.stub() - @EditorUpdatesController.listenForUpdatesFromDocumentUpdater() + describe("listenForUpdatesFromDocumentUpdater", function() { + beforeEach(function() { + this.rclient.length = 0; // clear any existing clients + this.EditorUpdatesController.rclientList = [this.redis.createClient("first"), this.redis.createClient("second")]; + this.rclient[0].subscribe = sinon.stub(); + this.rclient[0].on = sinon.stub(); + this.rclient[1].subscribe = sinon.stub(); + this.rclient[1].on = sinon.stub(); + return this.EditorUpdatesController.listenForUpdatesFromDocumentUpdater(); + }); - it "should subscribe to the doc-updater stream", -> - @rclient[0].subscribe.calledWith("applied-ops").should.equal true + it("should subscribe to the doc-updater stream", function() { + return this.rclient[0].subscribe.calledWith("applied-ops").should.equal(true); + }); - it "should register a callback to handle updates", -> - @rclient[0].on.calledWith("message").should.equal true + it("should register a callback to handle updates", function() { + return this.rclient[0].on.calledWith("message").should.equal(true); + }); - it "should subscribe to any additional doc-updater stream", -> - @rclient[1].subscribe.calledWith("applied-ops").should.equal true - @rclient[1].on.calledWith("message").should.equal true + return it("should subscribe to any additional doc-updater stream", function() { + this.rclient[1].subscribe.calledWith("applied-ops").should.equal(true); + return this.rclient[1].on.calledWith("message").should.equal(true); + }); + }); - describe "_processMessageFromDocumentUpdater", -> - describe "with bad JSON", -> - beforeEach -> - @SafeJsonParse.parse = sinon.stub().callsArgWith 1, new Error("oops") - @EditorUpdatesController._processMessageFromDocumentUpdater @io, "applied-ops", "blah" + describe("_processMessageFromDocumentUpdater", function() { + describe("with bad JSON", function() { + beforeEach(function() { + this.SafeJsonParse.parse = sinon.stub().callsArgWith(1, new Error("oops")); + return this.EditorUpdatesController._processMessageFromDocumentUpdater(this.io, "applied-ops", "blah"); + }); - it "should log an error", -> - @logger.error.called.should.equal true + return it("should log an error", function() { + return this.logger.error.called.should.equal(true); + }); + }); - describe "with update", -> - beforeEach -> - @message = - doc_id: @doc_id + describe("with update", function() { + beforeEach(function() { + this.message = { + doc_id: this.doc_id, op: {t: "foo", p: 12} - @EditorUpdatesController._applyUpdateFromDocumentUpdater = sinon.stub() - @EditorUpdatesController._processMessageFromDocumentUpdater @io, "applied-ops", JSON.stringify(@message) + }; + this.EditorUpdatesController._applyUpdateFromDocumentUpdater = sinon.stub(); + return this.EditorUpdatesController._processMessageFromDocumentUpdater(this.io, "applied-ops", JSON.stringify(this.message)); + }); - it "should apply the update", -> - @EditorUpdatesController._applyUpdateFromDocumentUpdater - .calledWith(@io, @doc_id, @message.op) - .should.equal true + return it("should apply the update", function() { + return this.EditorUpdatesController._applyUpdateFromDocumentUpdater + .calledWith(this.io, this.doc_id, this.message.op) + .should.equal(true); + }); + }); - describe "with error", -> - beforeEach -> - @message = - doc_id: @doc_id + return describe("with error", function() { + beforeEach(function() { + this.message = { + doc_id: this.doc_id, error: "Something went wrong" - @EditorUpdatesController._processErrorFromDocumentUpdater = sinon.stub() - @EditorUpdatesController._processMessageFromDocumentUpdater @io, "applied-ops", JSON.stringify(@message) + }; + this.EditorUpdatesController._processErrorFromDocumentUpdater = sinon.stub(); + return this.EditorUpdatesController._processMessageFromDocumentUpdater(this.io, "applied-ops", JSON.stringify(this.message)); + }); - it "should process the error", -> - @EditorUpdatesController._processErrorFromDocumentUpdater - .calledWith(@io, @doc_id, @message.error) - .should.equal true + return it("should process the error", function() { + return this.EditorUpdatesController._processErrorFromDocumentUpdater + .calledWith(this.io, this.doc_id, this.message.error) + .should.equal(true); + }); + }); + }); - describe "_applyUpdateFromDocumentUpdater", -> - beforeEach -> - @sourceClient = new MockClient() - @otherClients = [new MockClient(), new MockClient()] - @update = - op: [ t: "foo", p: 12 ] - meta: source: @sourceClient.publicId - v: @version = 42 - doc: @doc_id - @io.sockets = - clients: sinon.stub().returns([@sourceClient, @otherClients..., @sourceClient]) # include a duplicate client + describe("_applyUpdateFromDocumentUpdater", function() { + beforeEach(function() { + this.sourceClient = new MockClient(); + this.otherClients = [new MockClient(), new MockClient()]; + this.update = { + op: [ {t: "foo", p: 12} ], + meta: { source: this.sourceClient.publicId + }, + v: (this.version = 42), + doc: this.doc_id + }; + return this.io.sockets = + {clients: sinon.stub().returns([this.sourceClient, ...Array.from(this.otherClients), this.sourceClient])}; + }); // include a duplicate client - describe "normally", -> - beforeEach -> - @EditorUpdatesController._applyUpdateFromDocumentUpdater @io, @doc_id, @update + describe("normally", function() { + beforeEach(function() { + return this.EditorUpdatesController._applyUpdateFromDocumentUpdater(this.io, this.doc_id, this.update); + }); - it "should send a version bump to the source client", -> - @sourceClient.emit - .calledWith("otUpdateApplied", v: @version, doc: @doc_id) - .should.equal true - @sourceClient.emit.calledOnce.should.equal true + it("should send a version bump to the source client", function() { + this.sourceClient.emit + .calledWith("otUpdateApplied", {v: this.version, doc: this.doc_id}) + .should.equal(true); + return this.sourceClient.emit.calledOnce.should.equal(true); + }); - it "should get the clients connected to the document", -> - @io.sockets.clients - .calledWith(@doc_id) - .should.equal true + it("should get the clients connected to the document", function() { + return this.io.sockets.clients + .calledWith(this.doc_id) + .should.equal(true); + }); - it "should send the full update to the other clients", -> - for client in @otherClients + return it("should send the full update to the other clients", function() { + return Array.from(this.otherClients).map((client) => client.emit - .calledWith("otUpdateApplied", @update) - .should.equal true + .calledWith("otUpdateApplied", this.update) + .should.equal(true)); + }); + }); - describe "with a duplicate op", -> - beforeEach -> - @update.dup = true - @EditorUpdatesController._applyUpdateFromDocumentUpdater @io, @doc_id, @update + return describe("with a duplicate op", function() { + beforeEach(function() { + this.update.dup = true; + return this.EditorUpdatesController._applyUpdateFromDocumentUpdater(this.io, this.doc_id, this.update); + }); - it "should send a version bump to the source client as usual", -> - @sourceClient.emit - .calledWith("otUpdateApplied", v: @version, doc: @doc_id) - .should.equal true + it("should send a version bump to the source client as usual", function() { + return this.sourceClient.emit + .calledWith("otUpdateApplied", {v: this.version, doc: this.doc_id}) + .should.equal(true); + }); - it "should not send anything to the other clients (they've already had the op)", -> - for client in @otherClients + return it("should not send anything to the other clients (they've already had the op)", function() { + return Array.from(this.otherClients).map((client) => client.emit .calledWith("otUpdateApplied") - .should.equal false + .should.equal(false)); + }); + }); + }); - describe "_processErrorFromDocumentUpdater", -> - beforeEach -> - @clients = [new MockClient(), new MockClient()] - @io.sockets = - clients: sinon.stub().returns(@clients) - @EditorUpdatesController._processErrorFromDocumentUpdater @io, @doc_id, "Something went wrong" + return describe("_processErrorFromDocumentUpdater", function() { + beforeEach(function() { + this.clients = [new MockClient(), new MockClient()]; + this.io.sockets = + {clients: sinon.stub().returns(this.clients)}; + return this.EditorUpdatesController._processErrorFromDocumentUpdater(this.io, this.doc_id, "Something went wrong"); + }); - it "should log a warning", -> - @logger.warn.called.should.equal true + it("should log a warning", function() { + return this.logger.warn.called.should.equal(true); + }); - it "should disconnect all clients in that document", -> - @io.sockets.clients.calledWith(@doc_id).should.equal true - for client in @clients - client.disconnect.called.should.equal true + return it("should disconnect all clients in that document", function() { + this.io.sockets.clients.calledWith(this.doc_id).should.equal(true); + return Array.from(this.clients).map((client) => + client.disconnect.called.should.equal(true)); + }); + }); +}); diff --git a/services/real-time/test/unit/coffee/DocumentUpdaterManagerTests.js b/services/real-time/test/unit/coffee/DocumentUpdaterManagerTests.js index aa4600d757..c5117a2fc0 100644 --- a/services/real-time/test/unit/coffee/DocumentUpdaterManagerTests.js +++ b/services/real-time/test/unit/coffee/DocumentUpdaterManagerTests.js @@ -1,193 +1,260 @@ -require('chai').should() -sinon = require("sinon") -SandboxedModule = require('sandboxed-module') -path = require "path" -modulePath = '../../../app/js/DocumentUpdaterManager' +/* + * decaffeinate suggestions: + * DS102: Remove unnecessary code created because of implicit returns + * Full docs: https://github.com/decaffeinate/decaffeinate/blob/master/docs/suggestions.md + */ +require('chai').should(); +const sinon = require("sinon"); +const SandboxedModule = require('sandboxed-module'); +const path = require("path"); +const modulePath = '../../../app/js/DocumentUpdaterManager'; -describe 'DocumentUpdaterManager', -> - beforeEach -> - @project_id = "project-id-923" - @doc_id = "doc-id-394" - @lines = ["one", "two", "three"] - @version = 42 - @settings = - apis: documentupdater: url: "http://doc-updater.example.com" - redis: documentupdater: - key_schema: - pendingUpdates: ({doc_id}) -> "PendingUpdates:#{doc_id}" - maxUpdateSize: 7 * 1024 * 1024 - @rclient = {auth:->} - - @DocumentUpdaterManager = SandboxedModule.require modulePath, - requires: - 'settings-sharelatex':@settings - 'logger-sharelatex': @logger = {log: sinon.stub(), error: sinon.stub(), warn: sinon.stub()} - 'request': @request = {} - 'redis-sharelatex' : createClient: () => @rclient - 'metrics-sharelatex': @Metrics = - summary: sinon.stub() - Timer: class Timer - done: () -> - globals: - JSON: @JSON = Object.create(JSON) # avoid modifying JSON object directly - - describe "getDocument", -> - beforeEach -> - @callback = sinon.stub() - - describe "successfully", -> - beforeEach -> - @body = JSON.stringify - lines: @lines - version: @version - ops: @ops = ["mock-op-1", "mock-op-2"] - ranges: @ranges = {"mock": "ranges"} - @fromVersion = 2 - @request.get = sinon.stub().callsArgWith(1, null, {statusCode: 200}, @body) - @DocumentUpdaterManager.getDocument @project_id, @doc_id, @fromVersion, @callback - - it 'should get the document from the document updater', -> - url = "#{@settings.apis.documentupdater.url}/project/#{@project_id}/doc/#{@doc_id}?fromVersion=#{@fromVersion}" - @request.get.calledWith(url).should.equal true - - it "should call the callback with the lines, version, ranges and ops", -> - @callback.calledWith(null, @lines, @version, @ranges, @ops).should.equal true - - describe "when the document updater API returns an error", -> - beforeEach -> - @request.get = sinon.stub().callsArgWith(1, @error = new Error("something went wrong"), null, null) - @DocumentUpdaterManager.getDocument @project_id, @doc_id, @fromVersion, @callback - - it "should return an error to the callback", -> - @callback.calledWith(@error).should.equal true - - [404, 422].forEach (statusCode) -> - describe "when the document updater returns a #{statusCode} status code", -> - beforeEach -> - @request.get = sinon.stub().callsArgWith(1, null, { statusCode }, "") - @DocumentUpdaterManager.getDocument @project_id, @doc_id, @fromVersion, @callback - - it "should return the callback with an error", -> - @callback.called.should.equal(true) - err = @callback.getCall(0).args[0] - err.should.have.property('statusCode', statusCode) - err.should.have.property('message', "doc updater could not load requested ops") - @logger.error.called.should.equal(false) - @logger.warn.called.should.equal(true) - - describe "when the document updater returns a failure error code", -> - beforeEach -> - @request.get = sinon.stub().callsArgWith(1, null, { statusCode: 500 }, "") - @DocumentUpdaterManager.getDocument @project_id, @doc_id, @fromVersion, @callback - - it "should return the callback with an error", -> - @callback.called.should.equal(true) - err = @callback.getCall(0).args[0] - err.should.have.property('statusCode', 500) - err.should.have.property('message', "doc updater returned a non-success status code: 500") - @logger.error.called.should.equal(true) - - describe 'flushProjectToMongoAndDelete', -> - beforeEach -> - @callback = sinon.stub() - - describe "successfully", -> - beforeEach -> - @request.del = sinon.stub().callsArgWith(1, null, {statusCode: 204}, "") - @DocumentUpdaterManager.flushProjectToMongoAndDelete @project_id, @callback - - it 'should delete the project from the document updater', -> - url = "#{@settings.apis.documentupdater.url}/project/#{@project_id}?background=true" - @request.del.calledWith(url).should.equal true - - it "should call the callback with no error", -> - @callback.calledWith(null).should.equal true - - describe "when the document updater API returns an error", -> - beforeEach -> - @request.del = sinon.stub().callsArgWith(1, @error = new Error("something went wrong"), null, null) - @DocumentUpdaterManager.flushProjectToMongoAndDelete @project_id, @callback - - it "should return an error to the callback", -> - @callback.calledWith(@error).should.equal true - - describe "when the document updater returns a failure error code", -> - beforeEach -> - @request.del = sinon.stub().callsArgWith(1, null, { statusCode: 500 }, "") - @DocumentUpdaterManager.flushProjectToMongoAndDelete @project_id, @callback - - it "should return the callback with an error", -> - @callback.called.should.equal(true) - err = @callback.getCall(0).args[0] - err.should.have.property('statusCode', 500) - err.should.have.property('message', "document updater returned a failure status code: 500") - - describe 'queueChange', -> - beforeEach -> - @change = { - "doc":"1234567890", - "op":["d":"test", "p":345] - "v": 789 - } - @rclient.rpush = sinon.stub().yields() - @callback = sinon.stub() - - describe "successfully", -> - beforeEach -> - @DocumentUpdaterManager.queueChange(@project_id, @doc_id, @change, @callback) - - it "should push the change", -> - @rclient.rpush - .calledWith("PendingUpdates:#{@doc_id}", JSON.stringify(@change)) - .should.equal true - - it "should notify the doc updater of the change via the pending-updates-list queue", -> - @rclient.rpush - .calledWith("pending-updates-list", "#{@project_id}:#{@doc_id}") - .should.equal true - - describe "with error talking to redis during rpush", -> - beforeEach -> - @rclient.rpush = sinon.stub().yields(new Error("something went wrong")) - @DocumentUpdaterManager.queueChange(@project_id, @doc_id, @change, @callback) - - it "should return an error", -> - @callback.calledWithExactly(sinon.match(Error)).should.equal true - - describe "with null byte corruption", -> - beforeEach -> - @JSON.stringify = () -> return '["bad bytes! \u0000 <- here"]' - @DocumentUpdaterManager.queueChange(@project_id, @doc_id, @change, @callback) - - it "should return an error", -> - @callback.calledWithExactly(sinon.match(Error)).should.equal true - - it "should not push the change onto the pending-updates-list queue", -> - @rclient.rpush.called.should.equal false - - describe "when the update is too large", -> - beforeEach -> - @change = {op: {p: 12,t: "update is too large".repeat(1024 * 400)}} - @DocumentUpdaterManager.queueChange(@project_id, @doc_id, @change, @callback) - - it "should return an error", -> - @callback.calledWithExactly(sinon.match(Error)).should.equal true - - it "should add the size to the error", -> - @callback.args[0][0].updateSize.should.equal 7782422 - - it "should not push the change onto the pending-updates-list queue", -> - @rclient.rpush.called.should.equal false - - describe "with invalid keys", -> - beforeEach -> - @change = { - "op":["d":"test", "p":345] - "version": 789 # not a valid key +describe('DocumentUpdaterManager', function() { + beforeEach(function() { + let Timer; + this.project_id = "project-id-923"; + this.doc_id = "doc-id-394"; + this.lines = ["one", "two", "three"]; + this.version = 42; + this.settings = { + apis: { documentupdater: {url: "http://doc-updater.example.com"} + }, + redis: { documentupdater: { + key_schema: { + pendingUpdates({doc_id}) { return `PendingUpdates:${doc_id}`; } } - @DocumentUpdaterManager.queueChange(@project_id, @doc_id, @change, @callback) + } + }, + maxUpdateSize: 7 * 1024 * 1024 + }; + this.rclient = {auth() {}}; - it "should remove the invalid keys from the change", -> - @rclient.rpush - .calledWith("PendingUpdates:#{@doc_id}", JSON.stringify({op:@change.op})) - .should.equal true + return this.DocumentUpdaterManager = SandboxedModule.require(modulePath, { + requires: { + 'settings-sharelatex':this.settings, + 'logger-sharelatex': (this.logger = {log: sinon.stub(), error: sinon.stub(), warn: sinon.stub()}), + 'request': (this.request = {}), + 'redis-sharelatex' : { createClient: () => this.rclient + }, + 'metrics-sharelatex': (this.Metrics = { + summary: sinon.stub(), + Timer: (Timer = class Timer { + done() {} + }) + }) + }, + globals: { + JSON: (this.JSON = Object.create(JSON)) + } + } + ); + }); // avoid modifying JSON object directly + + describe("getDocument", function() { + beforeEach(function() { + return this.callback = sinon.stub(); + }); + + describe("successfully", function() { + beforeEach(function() { + this.body = JSON.stringify({ + lines: this.lines, + version: this.version, + ops: (this.ops = ["mock-op-1", "mock-op-2"]), + ranges: (this.ranges = {"mock": "ranges"})}); + this.fromVersion = 2; + this.request.get = sinon.stub().callsArgWith(1, null, {statusCode: 200}, this.body); + return this.DocumentUpdaterManager.getDocument(this.project_id, this.doc_id, this.fromVersion, this.callback); + }); + + it('should get the document from the document updater', function() { + const url = `${this.settings.apis.documentupdater.url}/project/${this.project_id}/doc/${this.doc_id}?fromVersion=${this.fromVersion}`; + return this.request.get.calledWith(url).should.equal(true); + }); + + return it("should call the callback with the lines, version, ranges and ops", function() { + return this.callback.calledWith(null, this.lines, this.version, this.ranges, this.ops).should.equal(true); + }); + }); + + describe("when the document updater API returns an error", function() { + beforeEach(function() { + this.request.get = sinon.stub().callsArgWith(1, (this.error = new Error("something went wrong")), null, null); + return this.DocumentUpdaterManager.getDocument(this.project_id, this.doc_id, this.fromVersion, this.callback); + }); + + return it("should return an error to the callback", function() { + return this.callback.calledWith(this.error).should.equal(true); + }); + }); + + [404, 422].forEach(statusCode => describe(`when the document updater returns a ${statusCode} status code`, function() { + beforeEach(function() { + this.request.get = sinon.stub().callsArgWith(1, null, { statusCode }, ""); + return this.DocumentUpdaterManager.getDocument(this.project_id, this.doc_id, this.fromVersion, this.callback); + }); + + return it("should return the callback with an error", function() { + this.callback.called.should.equal(true); + const err = this.callback.getCall(0).args[0]; + err.should.have.property('statusCode', statusCode); + err.should.have.property('message', "doc updater could not load requested ops"); + this.logger.error.called.should.equal(false); + return this.logger.warn.called.should.equal(true); + }); + })); + + return describe("when the document updater returns a failure error code", function() { + beforeEach(function() { + this.request.get = sinon.stub().callsArgWith(1, null, { statusCode: 500 }, ""); + return this.DocumentUpdaterManager.getDocument(this.project_id, this.doc_id, this.fromVersion, this.callback); + }); + + return it("should return the callback with an error", function() { + this.callback.called.should.equal(true); + const err = this.callback.getCall(0).args[0]; + err.should.have.property('statusCode', 500); + err.should.have.property('message', "doc updater returned a non-success status code: 500"); + return this.logger.error.called.should.equal(true); + }); + }); + }); + + describe('flushProjectToMongoAndDelete', function() { + beforeEach(function() { + return this.callback = sinon.stub(); + }); + + describe("successfully", function() { + beforeEach(function() { + this.request.del = sinon.stub().callsArgWith(1, null, {statusCode: 204}, ""); + return this.DocumentUpdaterManager.flushProjectToMongoAndDelete(this.project_id, this.callback); + }); + + it('should delete the project from the document updater', function() { + const url = `${this.settings.apis.documentupdater.url}/project/${this.project_id}?background=true`; + return this.request.del.calledWith(url).should.equal(true); + }); + + return it("should call the callback with no error", function() { + return this.callback.calledWith(null).should.equal(true); + }); + }); + + describe("when the document updater API returns an error", function() { + beforeEach(function() { + this.request.del = sinon.stub().callsArgWith(1, (this.error = new Error("something went wrong")), null, null); + return this.DocumentUpdaterManager.flushProjectToMongoAndDelete(this.project_id, this.callback); + }); + + return it("should return an error to the callback", function() { + return this.callback.calledWith(this.error).should.equal(true); + }); + }); + + return describe("when the document updater returns a failure error code", function() { + beforeEach(function() { + this.request.del = sinon.stub().callsArgWith(1, null, { statusCode: 500 }, ""); + return this.DocumentUpdaterManager.flushProjectToMongoAndDelete(this.project_id, this.callback); + }); + + return it("should return the callback with an error", function() { + this.callback.called.should.equal(true); + const err = this.callback.getCall(0).args[0]; + err.should.have.property('statusCode', 500); + return err.should.have.property('message', "document updater returned a failure status code: 500"); + }); + }); + }); + + return describe('queueChange', function() { + beforeEach(function() { + this.change = { + "doc":"1234567890", + "op":[{"d":"test", "p":345}], + "v": 789 + }; + this.rclient.rpush = sinon.stub().yields(); + return this.callback = sinon.stub(); + }); + + describe("successfully", function() { + beforeEach(function() { + return this.DocumentUpdaterManager.queueChange(this.project_id, this.doc_id, this.change, this.callback); + }); + + it("should push the change", function() { + return this.rclient.rpush + .calledWith(`PendingUpdates:${this.doc_id}`, JSON.stringify(this.change)) + .should.equal(true); + }); + + return it("should notify the doc updater of the change via the pending-updates-list queue", function() { + return this.rclient.rpush + .calledWith("pending-updates-list", `${this.project_id}:${this.doc_id}`) + .should.equal(true); + }); + }); + + describe("with error talking to redis during rpush", function() { + beforeEach(function() { + this.rclient.rpush = sinon.stub().yields(new Error("something went wrong")); + return this.DocumentUpdaterManager.queueChange(this.project_id, this.doc_id, this.change, this.callback); + }); + + return it("should return an error", function() { + return this.callback.calledWithExactly(sinon.match(Error)).should.equal(true); + }); + }); + + describe("with null byte corruption", function() { + beforeEach(function() { + this.JSON.stringify = () => '["bad bytes! \u0000 <- here"]'; + return this.DocumentUpdaterManager.queueChange(this.project_id, this.doc_id, this.change, this.callback); + }); + + it("should return an error", function() { + return this.callback.calledWithExactly(sinon.match(Error)).should.equal(true); + }); + + return it("should not push the change onto the pending-updates-list queue", function() { + return this.rclient.rpush.called.should.equal(false); + }); + }); + + describe("when the update is too large", function() { + beforeEach(function() { + this.change = {op: {p: 12,t: "update is too large".repeat(1024 * 400)}}; + return this.DocumentUpdaterManager.queueChange(this.project_id, this.doc_id, this.change, this.callback); + }); + + it("should return an error", function() { + return this.callback.calledWithExactly(sinon.match(Error)).should.equal(true); + }); + + it("should add the size to the error", function() { + return this.callback.args[0][0].updateSize.should.equal(7782422); + }); + + return it("should not push the change onto the pending-updates-list queue", function() { + return this.rclient.rpush.called.should.equal(false); + }); + }); + + return describe("with invalid keys", function() { + beforeEach(function() { + this.change = { + "op":[{"d":"test", "p":345}], + "version": 789 // not a valid key + }; + return this.DocumentUpdaterManager.queueChange(this.project_id, this.doc_id, this.change, this.callback); + }); + + return it("should remove the invalid keys from the change", function() { + return this.rclient.rpush + .calledWith(`PendingUpdates:${this.doc_id}`, JSON.stringify({op:this.change.op})) + .should.equal(true); + }); + }); + }); +}); diff --git a/services/real-time/test/unit/coffee/DrainManagerTests.js b/services/real-time/test/unit/coffee/DrainManagerTests.js index 88009f02cd..87bdaeb6d3 100644 --- a/services/real-time/test/unit/coffee/DrainManagerTests.js +++ b/services/real-time/test/unit/coffee/DrainManagerTests.js @@ -1,81 +1,113 @@ -should = require('chai').should() -sinon = require "sinon" -SandboxedModule = require('sandboxed-module') -path = require "path" -modulePath = path.join __dirname, "../../../app/js/DrainManager" +/* + * decaffeinate suggestions: + * DS102: Remove unnecessary code created because of implicit returns + * Full docs: https://github.com/decaffeinate/decaffeinate/blob/master/docs/suggestions.md + */ +const should = require('chai').should(); +const sinon = require("sinon"); +const SandboxedModule = require('sandboxed-module'); +const path = require("path"); +const modulePath = path.join(__dirname, "../../../app/js/DrainManager"); -describe "DrainManager", -> - beforeEach -> - @DrainManager = SandboxedModule.require modulePath, requires: - "logger-sharelatex": @logger = log: sinon.stub() - @io = - sockets: +describe("DrainManager", function() { + beforeEach(function() { + this.DrainManager = SandboxedModule.require(modulePath, { requires: { + "logger-sharelatex": (this.logger = {log: sinon.stub()}) + } + } + ); + return this.io = { + sockets: { clients: sinon.stub() + } + }; + }); - describe "startDrainTimeWindow", -> - beforeEach -> - @clients = [] - for i in [0..5399] - @clients[i] = { - id: i + describe("startDrainTimeWindow", function() { + beforeEach(function() { + this.clients = []; + for (let i = 0; i <= 5399; i++) { + this.clients[i] = { + id: i, emit: sinon.stub() - } - @io.sockets.clients.returns @clients - @DrainManager.startDrain = sinon.stub() + }; + } + this.io.sockets.clients.returns(this.clients); + return this.DrainManager.startDrain = sinon.stub(); + }); - it "should set a drain rate fast enough", (done)-> - @DrainManager.startDrainTimeWindow(@io, 9) - @DrainManager.startDrain.calledWith(@io, 10).should.equal true - done() + return it("should set a drain rate fast enough", function(done){ + this.DrainManager.startDrainTimeWindow(this.io, 9); + this.DrainManager.startDrain.calledWith(this.io, 10).should.equal(true); + return done(); + }); + }); - describe "reconnectNClients", -> - beforeEach -> - @clients = [] - for i in [0..9] - @clients[i] = { - id: i + return describe("reconnectNClients", function() { + beforeEach(function() { + this.clients = []; + for (let i = 0; i <= 9; i++) { + this.clients[i] = { + id: i, emit: sinon.stub() - } - @io.sockets.clients.returns @clients + }; + } + return this.io.sockets.clients.returns(this.clients); + }); - describe "after first pass", -> - beforeEach -> - @DrainManager.reconnectNClients(@io, 3) + return describe("after first pass", function() { + beforeEach(function() { + return this.DrainManager.reconnectNClients(this.io, 3); + }); - it "should reconnect the first 3 clients", -> - for i in [0..2] - @clients[i].emit.calledWith("reconnectGracefully").should.equal true + it("should reconnect the first 3 clients", function() { + return [0, 1, 2].map((i) => + this.clients[i].emit.calledWith("reconnectGracefully").should.equal(true)); + }); - it "should not reconnect any more clients", -> - for i in [3..9] - @clients[i].emit.calledWith("reconnectGracefully").should.equal false + it("should not reconnect any more clients", function() { + return [3, 4, 5, 6, 7, 8, 9].map((i) => + this.clients[i].emit.calledWith("reconnectGracefully").should.equal(false)); + }); - describe "after second pass", -> - beforeEach -> - @DrainManager.reconnectNClients(@io, 3) + return describe("after second pass", function() { + beforeEach(function() { + return this.DrainManager.reconnectNClients(this.io, 3); + }); - it "should reconnect the next 3 clients", -> - for i in [3..5] - @clients[i].emit.calledWith("reconnectGracefully").should.equal true + it("should reconnect the next 3 clients", function() { + return [3, 4, 5].map((i) => + this.clients[i].emit.calledWith("reconnectGracefully").should.equal(true)); + }); - it "should not reconnect any more clients", -> - for i in [6..9] - @clients[i].emit.calledWith("reconnectGracefully").should.equal false + it("should not reconnect any more clients", function() { + return [6, 7, 8, 9].map((i) => + this.clients[i].emit.calledWith("reconnectGracefully").should.equal(false)); + }); - it "should not reconnect the first 3 clients again", -> - for i in [0..2] - @clients[i].emit.calledOnce.should.equal true + it("should not reconnect the first 3 clients again", function() { + return [0, 1, 2].map((i) => + this.clients[i].emit.calledOnce.should.equal(true)); + }); - describe "after final pass", -> - beforeEach -> - @DrainManager.reconnectNClients(@io, 100) + return describe("after final pass", function() { + beforeEach(function() { + return this.DrainManager.reconnectNClients(this.io, 100); + }); - it "should not reconnect the first 6 clients again", -> - for i in [0..5] - @clients[i].emit.calledOnce.should.equal true + it("should not reconnect the first 6 clients again", function() { + return [0, 1, 2, 3, 4, 5].map((i) => + this.clients[i].emit.calledOnce.should.equal(true)); + }); - it "should log out that it reached the end", -> - @logger.log + return it("should log out that it reached the end", function() { + return this.logger.log .calledWith("All clients have been told to reconnectGracefully") - .should.equal true + .should.equal(true); + }); + }); + }); + }); + }); +}); diff --git a/services/real-time/test/unit/coffee/EventLoggerTests.js b/services/real-time/test/unit/coffee/EventLoggerTests.js index 93350af848..ab74861069 100644 --- a/services/real-time/test/unit/coffee/EventLoggerTests.js +++ b/services/real-time/test/unit/coffee/EventLoggerTests.js @@ -1,76 +1,101 @@ -require('chai').should() -expect = require("chai").expect -SandboxedModule = require('sandboxed-module') -modulePath = '../../../app/js/EventLogger' -sinon = require("sinon") -tk = require "timekeeper" +/* + * decaffeinate suggestions: + * DS102: Remove unnecessary code created because of implicit returns + * Full docs: https://github.com/decaffeinate/decaffeinate/blob/master/docs/suggestions.md + */ +require('chai').should(); +const { + expect +} = require("chai"); +const SandboxedModule = require('sandboxed-module'); +const modulePath = '../../../app/js/EventLogger'; +const sinon = require("sinon"); +const tk = require("timekeeper"); -describe 'EventLogger', -> - beforeEach -> - @start = Date.now() - tk.freeze(new Date(@start)) - @EventLogger = SandboxedModule.require modulePath, requires: - "logger-sharelatex": @logger = {error: sinon.stub(), warn: sinon.stub()} - "metrics-sharelatex": @metrics = {inc: sinon.stub()} - @channel = "applied-ops" - @id_1 = "random-hostname:abc-1" - @message_1 = "message-1" - @id_2 = "random-hostname:abc-2" - @message_2 = "message-2" +describe('EventLogger', function() { + beforeEach(function() { + this.start = Date.now(); + tk.freeze(new Date(this.start)); + this.EventLogger = SandboxedModule.require(modulePath, { requires: { + "logger-sharelatex": (this.logger = {error: sinon.stub(), warn: sinon.stub()}), + "metrics-sharelatex": (this.metrics = {inc: sinon.stub()}) + } + }); + this.channel = "applied-ops"; + this.id_1 = "random-hostname:abc-1"; + this.message_1 = "message-1"; + this.id_2 = "random-hostname:abc-2"; + return this.message_2 = "message-2"; + }); - afterEach -> - tk.reset() + afterEach(() => tk.reset()); - describe 'checkEventOrder', -> + return describe('checkEventOrder', function() { - describe 'when the events are in order', -> - beforeEach -> - @EventLogger.checkEventOrder(@channel, @id_1, @message_1) - @status = @EventLogger.checkEventOrder(@channel, @id_2, @message_2) + describe('when the events are in order', function() { + beforeEach(function() { + this.EventLogger.checkEventOrder(this.channel, this.id_1, this.message_1); + return this.status = this.EventLogger.checkEventOrder(this.channel, this.id_2, this.message_2); + }); - it 'should accept events in order', -> - expect(@status).to.be.undefined + it('should accept events in order', function() { + return expect(this.status).to.be.undefined; + }); - it 'should increment the valid event metric', -> - @metrics.inc.calledWith("event.#{@channel}.valid", 1) - .should.equal.true + return it('should increment the valid event metric', function() { + return this.metrics.inc.calledWith(`event.${this.channel}.valid`, 1) + .should.equal.true; + }); + }); - describe 'when there is a duplicate events', -> - beforeEach -> - @EventLogger.checkEventOrder(@channel, @id_1, @message_1) - @status = @EventLogger.checkEventOrder(@channel, @id_1, @message_1) + describe('when there is a duplicate events', function() { + beforeEach(function() { + this.EventLogger.checkEventOrder(this.channel, this.id_1, this.message_1); + return this.status = this.EventLogger.checkEventOrder(this.channel, this.id_1, this.message_1); + }); - it 'should return "duplicate" for the same event', -> - expect(@status).to.equal "duplicate" + it('should return "duplicate" for the same event', function() { + return expect(this.status).to.equal("duplicate"); + }); - it 'should increment the duplicate event metric', -> - @metrics.inc.calledWith("event.#{@channel}.duplicate", 1) - .should.equal.true + return it('should increment the duplicate event metric', function() { + return this.metrics.inc.calledWith(`event.${this.channel}.duplicate`, 1) + .should.equal.true; + }); + }); - describe 'when there are out of order events', -> - beforeEach -> - @EventLogger.checkEventOrder(@channel, @id_1, @message_1) - @EventLogger.checkEventOrder(@channel, @id_2, @message_2) - @status = @EventLogger.checkEventOrder(@channel, @id_1, @message_1) + describe('when there are out of order events', function() { + beforeEach(function() { + this.EventLogger.checkEventOrder(this.channel, this.id_1, this.message_1); + this.EventLogger.checkEventOrder(this.channel, this.id_2, this.message_2); + return this.status = this.EventLogger.checkEventOrder(this.channel, this.id_1, this.message_1); + }); - it 'should return "out-of-order" for the event', -> - expect(@status).to.equal "out-of-order" + it('should return "out-of-order" for the event', function() { + return expect(this.status).to.equal("out-of-order"); + }); - it 'should increment the out-of-order event metric', -> - @metrics.inc.calledWith("event.#{@channel}.out-of-order", 1) - .should.equal.true + return it('should increment the out-of-order event metric', function() { + return this.metrics.inc.calledWith(`event.${this.channel}.out-of-order`, 1) + .should.equal.true; + }); + }); - describe 'after MAX_STALE_TIME_IN_MS', -> - it 'should flush old entries', -> - @EventLogger.MAX_EVENTS_BEFORE_CLEAN = 10 - @EventLogger.checkEventOrder(@channel, @id_1, @message_1) - for i in [1..8] - status = @EventLogger.checkEventOrder(@channel, @id_1, @message_1) - expect(status).to.equal "duplicate" - # the next event should flush the old entries aboce - @EventLogger.MAX_STALE_TIME_IN_MS=1000 - tk.freeze(new Date(@start + 5 * 1000)) - # because we flushed the entries this should not be a duplicate - @EventLogger.checkEventOrder(@channel, 'other-1', @message_2) - status = @EventLogger.checkEventOrder(@channel, @id_1, @message_1) - expect(status).to.be.undefined \ No newline at end of file + return describe('after MAX_STALE_TIME_IN_MS', () => it('should flush old entries', function() { + let status; + this.EventLogger.MAX_EVENTS_BEFORE_CLEAN = 10; + this.EventLogger.checkEventOrder(this.channel, this.id_1, this.message_1); + for (let i = 1; i <= 8; i++) { + status = this.EventLogger.checkEventOrder(this.channel, this.id_1, this.message_1); + expect(status).to.equal("duplicate"); + } + // the next event should flush the old entries aboce + this.EventLogger.MAX_STALE_TIME_IN_MS=1000; + tk.freeze(new Date(this.start + (5 * 1000))); + // because we flushed the entries this should not be a duplicate + this.EventLogger.checkEventOrder(this.channel, 'other-1', this.message_2); + status = this.EventLogger.checkEventOrder(this.channel, this.id_1, this.message_1); + return expect(status).to.be.undefined; + })); + }); +}); \ No newline at end of file diff --git a/services/real-time/test/unit/coffee/RoomManagerTests.js b/services/real-time/test/unit/coffee/RoomManagerTests.js index c81663576d..63c25b3eae 100644 --- a/services/real-time/test/unit/coffee/RoomManagerTests.js +++ b/services/real-time/test/unit/coffee/RoomManagerTests.js @@ -1,288 +1,359 @@ -chai = require('chai') -expect = chai.expect -should = chai.should() -sinon = require("sinon") -modulePath = "../../../app/js/RoomManager.js" -SandboxedModule = require('sandboxed-module') +/* + * decaffeinate suggestions: + * DS102: Remove unnecessary code created because of implicit returns + * Full docs: https://github.com/decaffeinate/decaffeinate/blob/master/docs/suggestions.md + */ +const chai = require('chai'); +const { + expect +} = chai; +const should = chai.should(); +const sinon = require("sinon"); +const modulePath = "../../../app/js/RoomManager.js"; +const SandboxedModule = require('sandboxed-module'); -describe 'RoomManager', -> - beforeEach -> - @project_id = "project-id-123" - @doc_id = "doc-id-456" - @other_doc_id = "doc-id-789" - @client = {namespace: {name: ''}, id: "first-client"} - @RoomManager = SandboxedModule.require modulePath, requires: - "settings-sharelatex": @settings = {} - "logger-sharelatex": @logger = { log: sinon.stub(), warn: sinon.stub(), error: sinon.stub() } - "metrics-sharelatex": @metrics = { gauge: sinon.stub() } - @RoomManager._clientsInRoom = sinon.stub() - @RoomManager._clientAlreadyInRoom = sinon.stub() - @RoomEvents = @RoomManager.eventSource() - sinon.spy(@RoomEvents, 'emit') - sinon.spy(@RoomEvents, 'once') +describe('RoomManager', function() { + beforeEach(function() { + this.project_id = "project-id-123"; + this.doc_id = "doc-id-456"; + this.other_doc_id = "doc-id-789"; + this.client = {namespace: {name: ''}, id: "first-client"}; + this.RoomManager = SandboxedModule.require(modulePath, { requires: { + "settings-sharelatex": (this.settings = {}), + "logger-sharelatex": (this.logger = { log: sinon.stub(), warn: sinon.stub(), error: sinon.stub() }), + "metrics-sharelatex": (this.metrics = { gauge: sinon.stub() }) + } + }); + this.RoomManager._clientsInRoom = sinon.stub(); + this.RoomManager._clientAlreadyInRoom = sinon.stub(); + this.RoomEvents = this.RoomManager.eventSource(); + sinon.spy(this.RoomEvents, 'emit'); + return sinon.spy(this.RoomEvents, 'once'); + }); - describe "emitOnCompletion", -> - describe "when a subscribe errors", -> - afterEach () -> - process.removeListener("unhandledRejection", @onUnhandled) + describe("emitOnCompletion", () => describe("when a subscribe errors", function() { + afterEach(function() { + return process.removeListener("unhandledRejection", this.onUnhandled); + }); - beforeEach (done) -> - @onUnhandled = (error) => - @unhandledError = error - done(new Error("unhandledRejection: #{error.message}")) - process.on("unhandledRejection", @onUnhandled) + beforeEach(function(done) { + this.onUnhandled = error => { + this.unhandledError = error; + return done(new Error(`unhandledRejection: ${error.message}`)); + }; + process.on("unhandledRejection", this.onUnhandled); - reject = undefined - subscribePromise = new Promise((_, r) -> reject = r) - promises = [subscribePromise] - eventName = "project-subscribed-123" - @RoomEvents.once eventName, () -> - setTimeout(done, 100) - @RoomManager.emitOnCompletion(promises, eventName) - setTimeout(() -> reject(new Error("subscribe failed"))) + let reject = undefined; + const subscribePromise = new Promise((_, r) => reject = r); + const promises = [subscribePromise]; + const eventName = "project-subscribed-123"; + this.RoomEvents.once(eventName, () => setTimeout(done, 100)); + this.RoomManager.emitOnCompletion(promises, eventName); + return setTimeout(() => reject(new Error("subscribe failed"))); + }); - it "should keep going", () -> - expect(@unhandledError).to.not.exist + return it("should keep going", function() { + return expect(this.unhandledError).to.not.exist; + }); + })); - describe "joinProject", -> + describe("joinProject", function() { - describe "when the project room is empty", -> + describe("when the project room is empty", function() { - beforeEach (done) -> - @RoomManager._clientsInRoom - .withArgs(@client, @project_id) - .onFirstCall().returns(0) - @client.join = sinon.stub() - @callback = sinon.stub() - @RoomEvents.on 'project-active', (id) => - setTimeout () => - @RoomEvents.emit "project-subscribed-#{id}" - , 100 - @RoomManager.joinProject @client, @project_id, (err) => - @callback(err) - done() + beforeEach(function(done) { + this.RoomManager._clientsInRoom + .withArgs(this.client, this.project_id) + .onFirstCall().returns(0); + this.client.join = sinon.stub(); + this.callback = sinon.stub(); + this.RoomEvents.on('project-active', id => { + return setTimeout(() => { + return this.RoomEvents.emit(`project-subscribed-${id}`); + } + , 100); + }); + return this.RoomManager.joinProject(this.client, this.project_id, err => { + this.callback(err); + return done(); + }); + }); - it "should emit a 'project-active' event with the id", -> - @RoomEvents.emit.calledWithExactly('project-active', @project_id).should.equal true + it("should emit a 'project-active' event with the id", function() { + return this.RoomEvents.emit.calledWithExactly('project-active', this.project_id).should.equal(true); + }); - it "should listen for the 'project-subscribed-id' event", -> - @RoomEvents.once.calledWith("project-subscribed-#{@project_id}").should.equal true + it("should listen for the 'project-subscribed-id' event", function() { + return this.RoomEvents.once.calledWith(`project-subscribed-${this.project_id}`).should.equal(true); + }); - it "should join the room using the id", -> - @client.join.calledWithExactly(@project_id).should.equal true + return it("should join the room using the id", function() { + return this.client.join.calledWithExactly(this.project_id).should.equal(true); + }); + }); - describe "when there are other clients in the project room", -> + return describe("when there are other clients in the project room", function() { - beforeEach -> - @RoomManager._clientsInRoom - .withArgs(@client, @project_id) + beforeEach(function() { + this.RoomManager._clientsInRoom + .withArgs(this.client, this.project_id) .onFirstCall().returns(123) - .onSecondCall().returns(124) - @client.join = sinon.stub() - @RoomManager.joinProject @client, @project_id + .onSecondCall().returns(124); + this.client.join = sinon.stub(); + return this.RoomManager.joinProject(this.client, this.project_id); + }); - it "should join the room using the id", -> - @client.join.called.should.equal true + it("should join the room using the id", function() { + return this.client.join.called.should.equal(true); + }); - it "should not emit any events", -> - @RoomEvents.emit.called.should.equal false + return it("should not emit any events", function() { + return this.RoomEvents.emit.called.should.equal(false); + }); + }); + }); - describe "joinDoc", -> + describe("joinDoc", function() { - describe "when the doc room is empty", -> + describe("when the doc room is empty", function() { - beforeEach (done) -> - @RoomManager._clientsInRoom - .withArgs(@client, @doc_id) - .onFirstCall().returns(0) - @client.join = sinon.stub() - @callback = sinon.stub() - @RoomEvents.on 'doc-active', (id) => - setTimeout () => - @RoomEvents.emit "doc-subscribed-#{id}" - , 100 - @RoomManager.joinDoc @client, @doc_id, (err) => - @callback(err) - done() + beforeEach(function(done) { + this.RoomManager._clientsInRoom + .withArgs(this.client, this.doc_id) + .onFirstCall().returns(0); + this.client.join = sinon.stub(); + this.callback = sinon.stub(); + this.RoomEvents.on('doc-active', id => { + return setTimeout(() => { + return this.RoomEvents.emit(`doc-subscribed-${id}`); + } + , 100); + }); + return this.RoomManager.joinDoc(this.client, this.doc_id, err => { + this.callback(err); + return done(); + }); + }); - it "should emit a 'doc-active' event with the id", -> - @RoomEvents.emit.calledWithExactly('doc-active', @doc_id).should.equal true + it("should emit a 'doc-active' event with the id", function() { + return this.RoomEvents.emit.calledWithExactly('doc-active', this.doc_id).should.equal(true); + }); - it "should listen for the 'doc-subscribed-id' event", -> - @RoomEvents.once.calledWith("doc-subscribed-#{@doc_id}").should.equal true + it("should listen for the 'doc-subscribed-id' event", function() { + return this.RoomEvents.once.calledWith(`doc-subscribed-${this.doc_id}`).should.equal(true); + }); - it "should join the room using the id", -> - @client.join.calledWithExactly(@doc_id).should.equal true + return it("should join the room using the id", function() { + return this.client.join.calledWithExactly(this.doc_id).should.equal(true); + }); + }); - describe "when there are other clients in the doc room", -> + return describe("when there are other clients in the doc room", function() { - beforeEach -> - @RoomManager._clientsInRoom - .withArgs(@client, @doc_id) + beforeEach(function() { + this.RoomManager._clientsInRoom + .withArgs(this.client, this.doc_id) .onFirstCall().returns(123) - .onSecondCall().returns(124) - @client.join = sinon.stub() - @RoomManager.joinDoc @client, @doc_id + .onSecondCall().returns(124); + this.client.join = sinon.stub(); + return this.RoomManager.joinDoc(this.client, this.doc_id); + }); - it "should join the room using the id", -> - @client.join.called.should.equal true + it("should join the room using the id", function() { + return this.client.join.called.should.equal(true); + }); - it "should not emit any events", -> - @RoomEvents.emit.called.should.equal false + return it("should not emit any events", function() { + return this.RoomEvents.emit.called.should.equal(false); + }); + }); + }); - describe "leaveDoc", -> + describe("leaveDoc", function() { - describe "when doc room will be empty after this client has left", -> + describe("when doc room will be empty after this client has left", function() { - beforeEach -> - @RoomManager._clientAlreadyInRoom - .withArgs(@client, @doc_id) - .returns(true) - @RoomManager._clientsInRoom - .withArgs(@client, @doc_id) - .onCall(0).returns(0) - @client.leave = sinon.stub() - @RoomManager.leaveDoc @client, @doc_id + beforeEach(function() { + this.RoomManager._clientAlreadyInRoom + .withArgs(this.client, this.doc_id) + .returns(true); + this.RoomManager._clientsInRoom + .withArgs(this.client, this.doc_id) + .onCall(0).returns(0); + this.client.leave = sinon.stub(); + return this.RoomManager.leaveDoc(this.client, this.doc_id); + }); - it "should leave the room using the id", -> - @client.leave.calledWithExactly(@doc_id).should.equal true + it("should leave the room using the id", function() { + return this.client.leave.calledWithExactly(this.doc_id).should.equal(true); + }); - it "should emit a 'doc-empty' event with the id", -> - @RoomEvents.emit.calledWithExactly('doc-empty', @doc_id).should.equal true + return it("should emit a 'doc-empty' event with the id", function() { + return this.RoomEvents.emit.calledWithExactly('doc-empty', this.doc_id).should.equal(true); + }); + }); - describe "when there are other clients in the doc room", -> + describe("when there are other clients in the doc room", function() { - beforeEach -> - @RoomManager._clientAlreadyInRoom - .withArgs(@client, @doc_id) - .returns(true) - @RoomManager._clientsInRoom - .withArgs(@client, @doc_id) - .onCall(0).returns(123) - @client.leave = sinon.stub() - @RoomManager.leaveDoc @client, @doc_id + beforeEach(function() { + this.RoomManager._clientAlreadyInRoom + .withArgs(this.client, this.doc_id) + .returns(true); + this.RoomManager._clientsInRoom + .withArgs(this.client, this.doc_id) + .onCall(0).returns(123); + this.client.leave = sinon.stub(); + return this.RoomManager.leaveDoc(this.client, this.doc_id); + }); - it "should leave the room using the id", -> - @client.leave.calledWithExactly(@doc_id).should.equal true + it("should leave the room using the id", function() { + return this.client.leave.calledWithExactly(this.doc_id).should.equal(true); + }); - it "should not emit any events", -> - @RoomEvents.emit.called.should.equal false + return it("should not emit any events", function() { + return this.RoomEvents.emit.called.should.equal(false); + }); + }); - describe "when the client is not in the doc room", -> + return describe("when the client is not in the doc room", function() { - beforeEach -> - @RoomManager._clientAlreadyInRoom - .withArgs(@client, @doc_id) - .returns(false) - @RoomManager._clientsInRoom - .withArgs(@client, @doc_id) - .onCall(0).returns(0) - @client.leave = sinon.stub() - @RoomManager.leaveDoc @client, @doc_id + beforeEach(function() { + this.RoomManager._clientAlreadyInRoom + .withArgs(this.client, this.doc_id) + .returns(false); + this.RoomManager._clientsInRoom + .withArgs(this.client, this.doc_id) + .onCall(0).returns(0); + this.client.leave = sinon.stub(); + return this.RoomManager.leaveDoc(this.client, this.doc_id); + }); - it "should not leave the room", -> - @client.leave.called.should.equal false + it("should not leave the room", function() { + return this.client.leave.called.should.equal(false); + }); - it "should not emit any events", -> - @RoomEvents.emit.called.should.equal false + return it("should not emit any events", function() { + return this.RoomEvents.emit.called.should.equal(false); + }); + }); + }); - describe "leaveProjectAndDocs", -> + return describe("leaveProjectAndDocs", () => describe("when the client is connected to the project and multiple docs", function() { - describe "when the client is connected to the project and multiple docs", -> + beforeEach(function() { + this.RoomManager._roomsClientIsIn = sinon.stub().returns([this.project_id, this.doc_id, this.other_doc_id]); + this.client.join = sinon.stub(); + return this.client.leave = sinon.stub(); + }); - beforeEach -> - @RoomManager._roomsClientIsIn = sinon.stub().returns [@project_id, @doc_id, @other_doc_id] - @client.join = sinon.stub() - @client.leave = sinon.stub() + describe("when this is the only client connected", function() { - describe "when this is the only client connected", -> + beforeEach(function(done) { + // first call is for the join, + // second for the leave + this.RoomManager._clientsInRoom + .withArgs(this.client, this.doc_id) + .onCall(0).returns(0) + .onCall(1).returns(0); + this.RoomManager._clientsInRoom + .withArgs(this.client, this.other_doc_id) + .onCall(0).returns(0) + .onCall(1).returns(0); + this.RoomManager._clientsInRoom + .withArgs(this.client, this.project_id) + .onCall(0).returns(0) + .onCall(1).returns(0); + this.RoomManager._clientAlreadyInRoom + .withArgs(this.client, this.doc_id) + .returns(true) + .withArgs(this.client, this.other_doc_id) + .returns(true) + .withArgs(this.client, this.project_id) + .returns(true); + this.RoomEvents.on('project-active', id => { + return setTimeout(() => { + return this.RoomEvents.emit(`project-subscribed-${id}`); + } + , 100); + }); + this.RoomEvents.on('doc-active', id => { + return setTimeout(() => { + return this.RoomEvents.emit(`doc-subscribed-${id}`); + } + , 100); + }); + // put the client in the rooms + return this.RoomManager.joinProject(this.client, this.project_id, () => { + return this.RoomManager.joinDoc(this.client, this.doc_id, () => { + return this.RoomManager.joinDoc(this.client, this.other_doc_id, () => { + // now leave the project + this.RoomManager.leaveProjectAndDocs(this.client); + return done(); + }); + }); + }); + }); - beforeEach (done) -> - # first call is for the join, - # second for the leave - @RoomManager._clientsInRoom - .withArgs(@client, @doc_id) - .onCall(0).returns(0) - .onCall(1).returns(0) - @RoomManager._clientsInRoom - .withArgs(@client, @other_doc_id) - .onCall(0).returns(0) - .onCall(1).returns(0) - @RoomManager._clientsInRoom - .withArgs(@client, @project_id) - .onCall(0).returns(0) - .onCall(1).returns(0) - @RoomManager._clientAlreadyInRoom - .withArgs(@client, @doc_id) - .returns(true) - .withArgs(@client, @other_doc_id) - .returns(true) - .withArgs(@client, @project_id) - .returns(true) - @RoomEvents.on 'project-active', (id) => - setTimeout () => - @RoomEvents.emit "project-subscribed-#{id}" - , 100 - @RoomEvents.on 'doc-active', (id) => - setTimeout () => - @RoomEvents.emit "doc-subscribed-#{id}" - , 100 - # put the client in the rooms - @RoomManager.joinProject @client, @project_id, () => - @RoomManager.joinDoc @client, @doc_id, () => - @RoomManager.joinDoc @client, @other_doc_id, () => - # now leave the project - @RoomManager.leaveProjectAndDocs @client - done() + it("should leave all the docs", function() { + this.client.leave.calledWithExactly(this.doc_id).should.equal(true); + return this.client.leave.calledWithExactly(this.other_doc_id).should.equal(true); + }); - it "should leave all the docs", -> - @client.leave.calledWithExactly(@doc_id).should.equal true - @client.leave.calledWithExactly(@other_doc_id).should.equal true + it("should leave the project", function() { + return this.client.leave.calledWithExactly(this.project_id).should.equal(true); + }); - it "should leave the project", -> - @client.leave.calledWithExactly(@project_id).should.equal true + it("should emit a 'doc-empty' event with the id for each doc", function() { + this.RoomEvents.emit.calledWithExactly('doc-empty', this.doc_id).should.equal(true); + return this.RoomEvents.emit.calledWithExactly('doc-empty', this.other_doc_id).should.equal(true); + }); - it "should emit a 'doc-empty' event with the id for each doc", -> - @RoomEvents.emit.calledWithExactly('doc-empty', @doc_id).should.equal true - @RoomEvents.emit.calledWithExactly('doc-empty', @other_doc_id).should.equal true + return it("should emit a 'project-empty' event with the id for the project", function() { + return this.RoomEvents.emit.calledWithExactly('project-empty', this.project_id).should.equal(true); + }); + }); - it "should emit a 'project-empty' event with the id for the project", -> - @RoomEvents.emit.calledWithExactly('project-empty', @project_id).should.equal true + return describe("when other clients are still connected", function() { - describe "when other clients are still connected", -> + beforeEach(function() { + this.RoomManager._clientsInRoom + .withArgs(this.client, this.doc_id) + .onFirstCall().returns(123) + .onSecondCall().returns(122); + this.RoomManager._clientsInRoom + .withArgs(this.client, this.other_doc_id) + .onFirstCall().returns(123) + .onSecondCall().returns(122); + this.RoomManager._clientsInRoom + .withArgs(this.client, this.project_id) + .onFirstCall().returns(123) + .onSecondCall().returns(122); + this.RoomManager._clientAlreadyInRoom + .withArgs(this.client, this.doc_id) + .returns(true) + .withArgs(this.client, this.other_doc_id) + .returns(true) + .withArgs(this.client, this.project_id) + .returns(true); + return this.RoomManager.leaveProjectAndDocs(this.client); + }); - beforeEach -> - @RoomManager._clientsInRoom - .withArgs(@client, @doc_id) - .onFirstCall().returns(123) - .onSecondCall().returns(122) - @RoomManager._clientsInRoom - .withArgs(@client, @other_doc_id) - .onFirstCall().returns(123) - .onSecondCall().returns(122) - @RoomManager._clientsInRoom - .withArgs(@client, @project_id) - .onFirstCall().returns(123) - .onSecondCall().returns(122) - @RoomManager._clientAlreadyInRoom - .withArgs(@client, @doc_id) - .returns(true) - .withArgs(@client, @other_doc_id) - .returns(true) - .withArgs(@client, @project_id) - .returns(true) - @RoomManager.leaveProjectAndDocs @client + it("should leave all the docs", function() { + this.client.leave.calledWithExactly(this.doc_id).should.equal(true); + return this.client.leave.calledWithExactly(this.other_doc_id).should.equal(true); + }); - it "should leave all the docs", -> - @client.leave.calledWithExactly(@doc_id).should.equal true - @client.leave.calledWithExactly(@other_doc_id).should.equal true + it("should leave the project", function() { + return this.client.leave.calledWithExactly(this.project_id).should.equal(true); + }); - it "should leave the project", -> - @client.leave.calledWithExactly(@project_id).should.equal true - - it "should not emit any events", -> - @RoomEvents.emit.called.should.equal false \ No newline at end of file + return it("should not emit any events", function() { + return this.RoomEvents.emit.called.should.equal(false); + }); + }); + })); +}); \ No newline at end of file diff --git a/services/real-time/test/unit/coffee/SafeJsonParseTest.js b/services/real-time/test/unit/coffee/SafeJsonParseTest.js index b652a2faae..f417513e47 100644 --- a/services/real-time/test/unit/coffee/SafeJsonParseTest.js +++ b/services/real-time/test/unit/coffee/SafeJsonParseTest.js @@ -1,34 +1,51 @@ -require('chai').should() -expect = require("chai").expect -SandboxedModule = require('sandboxed-module') -modulePath = '../../../app/js/SafeJsonParse' -sinon = require("sinon") +/* + * decaffeinate suggestions: + * DS102: Remove unnecessary code created because of implicit returns + * Full docs: https://github.com/decaffeinate/decaffeinate/blob/master/docs/suggestions.md + */ +require('chai').should(); +const { + expect +} = require("chai"); +const SandboxedModule = require('sandboxed-module'); +const modulePath = '../../../app/js/SafeJsonParse'; +const sinon = require("sinon"); -describe 'SafeJsonParse', -> - beforeEach -> - @SafeJsonParse = SandboxedModule.require modulePath, requires: - "settings-sharelatex": @Settings = { +describe('SafeJsonParse', function() { + beforeEach(function() { + return this.SafeJsonParse = SandboxedModule.require(modulePath, { requires: { + "settings-sharelatex": (this.Settings = { maxUpdateSize: 16 * 1024 - } - "logger-sharelatex": @logger = {error: sinon.stub()} + }), + "logger-sharelatex": (this.logger = {error: sinon.stub()}) + } + });}); - describe "parse", -> - it "should parse documents correctly", (done) -> - @SafeJsonParse.parse '{"foo": "bar"}', (error, parsed) -> - expect(parsed).to.deep.equal {foo: "bar"} - done() + return describe("parse", function() { + it("should parse documents correctly", function(done) { + return this.SafeJsonParse.parse('{"foo": "bar"}', function(error, parsed) { + expect(parsed).to.deep.equal({foo: "bar"}); + return done(); + }); + }); - it "should return an error on bad data", (done) -> - @SafeJsonParse.parse 'blah', (error, parsed) -> - expect(error).to.exist - done() + it("should return an error on bad data", function(done) { + return this.SafeJsonParse.parse('blah', function(error, parsed) { + expect(error).to.exist; + return done(); + }); + }); - it "should return an error on oversized data", (done) -> - # we have a 2k overhead on top of max size - big_blob = Array(16*1024).join("A") - data = "{\"foo\": \"#{big_blob}\"}" - @Settings.maxUpdateSize = 2 * 1024 - @SafeJsonParse.parse data, (error, parsed) => - @logger.error.called.should.equal true - expect(error).to.exist - done() \ No newline at end of file + return it("should return an error on oversized data", function(done) { + // we have a 2k overhead on top of max size + const big_blob = Array(16*1024).join("A"); + const data = `{\"foo\": \"${big_blob}\"}`; + this.Settings.maxUpdateSize = 2 * 1024; + return this.SafeJsonParse.parse(data, (error, parsed) => { + this.logger.error.called.should.equal(true); + expect(error).to.exist; + return done(); + }); + }); + }); +}); \ No newline at end of file diff --git a/services/real-time/test/unit/coffee/SessionSocketsTests.js b/services/real-time/test/unit/coffee/SessionSocketsTests.js index 2f81699309..d85be502a7 100644 --- a/services/real-time/test/unit/coffee/SessionSocketsTests.js +++ b/services/real-time/test/unit/coffee/SessionSocketsTests.js @@ -1,126 +1,170 @@ -{EventEmitter} = require('events') -{expect} = require('chai') -SandboxedModule = require('sandboxed-module') -modulePath = '../../../app/js/SessionSockets' -sinon = require('sinon') +/* + * decaffeinate suggestions: + * DS102: Remove unnecessary code created because of implicit returns + * Full docs: https://github.com/decaffeinate/decaffeinate/blob/master/docs/suggestions.md + */ +const {EventEmitter} = require('events'); +const {expect} = require('chai'); +const SandboxedModule = require('sandboxed-module'); +const modulePath = '../../../app/js/SessionSockets'; +const sinon = require('sinon'); -describe 'SessionSockets', -> - before -> - @SessionSocketsModule = SandboxedModule.require modulePath - @io = new EventEmitter() - @id1 = Math.random().toString() - @id2 = Math.random().toString() - redisResponses = - error: [new Error('Redis: something went wrong'), null] +describe('SessionSockets', function() { + before(function() { + this.SessionSocketsModule = SandboxedModule.require(modulePath); + this.io = new EventEmitter(); + this.id1 = Math.random().toString(); + this.id2 = Math.random().toString(); + const redisResponses = { + error: [new Error('Redis: something went wrong'), null], unknownId: [null, null] - redisResponses[@id1] = [null, {user: {_id: '123'}}] - redisResponses[@id2] = [null, {user: {_id: 'abc'}}] + }; + redisResponses[this.id1] = [null, {user: {_id: '123'}}]; + redisResponses[this.id2] = [null, {user: {_id: 'abc'}}]; - @sessionStore = - get: sinon.stub().callsFake (id, fn) -> - fn.apply(null, redisResponses[id]) - @cookieParser = (req, res, next) -> - req.signedCookies = req._signedCookies - next() - @SessionSockets = @SessionSocketsModule(@io, @sessionStore, @cookieParser, 'ol.sid') - @checkSocket = (socket, fn) => - @SessionSockets.once('connection', fn) - @io.emit('connection', socket) + this.sessionStore = { + get: sinon.stub().callsFake((id, fn) => fn.apply(null, redisResponses[id])) + }; + this.cookieParser = function(req, res, next) { + req.signedCookies = req._signedCookies; + return next(); + }; + this.SessionSockets = this.SessionSocketsModule(this.io, this.sessionStore, this.cookieParser, 'ol.sid'); + return this.checkSocket = (socket, fn) => { + this.SessionSockets.once('connection', fn); + return this.io.emit('connection', socket); + }; + }); - describe 'without cookies', -> - before -> - @socket = {handshake: {}} + describe('without cookies', function() { + before(function() { + return this.socket = {handshake: {}};}); - it 'should return a lookup error', (done) -> - @checkSocket @socket, (error) -> - expect(error).to.exist - expect(error.message).to.equal('could not look up session by key') - done() + it('should return a lookup error', function(done) { + return this.checkSocket(this.socket, function(error) { + expect(error).to.exist; + expect(error.message).to.equal('could not look up session by key'); + return done(); + }); + }); - it 'should not query redis', (done) -> - @checkSocket @socket, () => - expect(@sessionStore.get.called).to.equal(false) - done() + return it('should not query redis', function(done) { + return this.checkSocket(this.socket, () => { + expect(this.sessionStore.get.called).to.equal(false); + return done(); + }); + }); + }); - describe 'with a different cookie', -> - before -> - @socket = {handshake: {_signedCookies: {other: 1}}} + describe('with a different cookie', function() { + before(function() { + return this.socket = {handshake: {_signedCookies: {other: 1}}};}); - it 'should return a lookup error', (done) -> - @checkSocket @socket, (error) -> - expect(error).to.exist - expect(error.message).to.equal('could not look up session by key') - done() + it('should return a lookup error', function(done) { + return this.checkSocket(this.socket, function(error) { + expect(error).to.exist; + expect(error.message).to.equal('could not look up session by key'); + return done(); + }); + }); - it 'should not query redis', (done) -> - @checkSocket @socket, () => - expect(@sessionStore.get.called).to.equal(false) - done() + return it('should not query redis', function(done) { + return this.checkSocket(this.socket, () => { + expect(this.sessionStore.get.called).to.equal(false); + return done(); + }); + }); + }); - describe 'with a valid cookie and a failing session lookup', -> - before -> - @socket = {handshake: {_signedCookies: {'ol.sid': 'error'}}} + describe('with a valid cookie and a failing session lookup', function() { + before(function() { + return this.socket = {handshake: {_signedCookies: {'ol.sid': 'error'}}};}); - it 'should query redis', (done) -> - @checkSocket @socket, () => - expect(@sessionStore.get.called).to.equal(true) - done() + it('should query redis', function(done) { + return this.checkSocket(this.socket, () => { + expect(this.sessionStore.get.called).to.equal(true); + return done(); + }); + }); - it 'should return a redis error', (done) -> - @checkSocket @socket, (error) -> - expect(error).to.exist - expect(error.message).to.equal('Redis: something went wrong') - done() + return it('should return a redis error', function(done) { + return this.checkSocket(this.socket, function(error) { + expect(error).to.exist; + expect(error.message).to.equal('Redis: something went wrong'); + return done(); + }); + }); + }); - describe 'with a valid cookie and no matching session', -> - before -> - @socket = {handshake: {_signedCookies: {'ol.sid': 'unknownId'}}} + describe('with a valid cookie and no matching session', function() { + before(function() { + return this.socket = {handshake: {_signedCookies: {'ol.sid': 'unknownId'}}};}); - it 'should query redis', (done) -> - @checkSocket @socket, () => - expect(@sessionStore.get.called).to.equal(true) - done() + it('should query redis', function(done) { + return this.checkSocket(this.socket, () => { + expect(this.sessionStore.get.called).to.equal(true); + return done(); + }); + }); - it 'should return a lookup error', (done) -> - @checkSocket @socket, (error) -> - expect(error).to.exist - expect(error.message).to.equal('could not look up session by key') - done() + return it('should return a lookup error', function(done) { + return this.checkSocket(this.socket, function(error) { + expect(error).to.exist; + expect(error.message).to.equal('could not look up session by key'); + return done(); + }); + }); + }); - describe 'with a valid cookie and a matching session', -> - before -> - @socket = {handshake: {_signedCookies: {'ol.sid': @id1}}} + describe('with a valid cookie and a matching session', function() { + before(function() { + return this.socket = {handshake: {_signedCookies: {'ol.sid': this.id1}}};}); - it 'should query redis', (done) -> - @checkSocket @socket, () => - expect(@sessionStore.get.called).to.equal(true) - done() + it('should query redis', function(done) { + return this.checkSocket(this.socket, () => { + expect(this.sessionStore.get.called).to.equal(true); + return done(); + }); + }); - it 'should not return an error', (done) -> - @checkSocket @socket, (error) -> - expect(error).to.not.exist - done() + it('should not return an error', function(done) { + return this.checkSocket(this.socket, function(error) { + expect(error).to.not.exist; + return done(); + }); + }); - it 'should return the session', (done) -> - @checkSocket @socket, (error, s, session) -> - expect(session).to.deep.equal({user: {_id: '123'}}) - done() + return it('should return the session', function(done) { + return this.checkSocket(this.socket, function(error, s, session) { + expect(session).to.deep.equal({user: {_id: '123'}}); + return done(); + }); + }); + }); - describe 'with a different valid cookie and matching session', -> - before -> - @socket = {handshake: {_signedCookies: {'ol.sid': @id2}}} + return describe('with a different valid cookie and matching session', function() { + before(function() { + return this.socket = {handshake: {_signedCookies: {'ol.sid': this.id2}}};}); - it 'should query redis', (done) -> - @checkSocket @socket, () => - expect(@sessionStore.get.called).to.equal(true) - done() + it('should query redis', function(done) { + return this.checkSocket(this.socket, () => { + expect(this.sessionStore.get.called).to.equal(true); + return done(); + }); + }); - it 'should not return an error', (done) -> - @checkSocket @socket, (error) -> - expect(error).to.not.exist - done() + it('should not return an error', function(done) { + return this.checkSocket(this.socket, function(error) { + expect(error).to.not.exist; + return done(); + }); + }); - it 'should return the other session', (done) -> - @checkSocket @socket, (error, s, session) -> - expect(session).to.deep.equal({user: {_id: 'abc'}}) - done() + return it('should return the other session', function(done) { + return this.checkSocket(this.socket, function(error, s, session) { + expect(session).to.deep.equal({user: {_id: 'abc'}}); + return done(); + }); + }); + }); +}); diff --git a/services/real-time/test/unit/coffee/WebApiManagerTests.js b/services/real-time/test/unit/coffee/WebApiManagerTests.js index e65ba93859..19d2bbe444 100644 --- a/services/real-time/test/unit/coffee/WebApiManagerTests.js +++ b/services/real-time/test/unit/coffee/WebApiManagerTests.js @@ -1,84 +1,111 @@ -chai = require('chai') -should = chai.should() -sinon = require("sinon") -modulePath = "../../../app/js/WebApiManager.js" -SandboxedModule = require('sandboxed-module') -{ CodedError } = require('../../../app/js/Errors') +/* + * decaffeinate suggestions: + * DS102: Remove unnecessary code created because of implicit returns + * Full docs: https://github.com/decaffeinate/decaffeinate/blob/master/docs/suggestions.md + */ +const chai = require('chai'); +const should = chai.should(); +const sinon = require("sinon"); +const modulePath = "../../../app/js/WebApiManager.js"; +const SandboxedModule = require('sandboxed-module'); +const { CodedError } = require('../../../app/js/Errors'); -describe 'WebApiManager', -> - beforeEach -> - @project_id = "project-id-123" - @user_id = "user-id-123" - @user = {_id: @user_id} - @callback = sinon.stub() - @WebApiManager = SandboxedModule.require modulePath, requires: - "request": @request = {} - "settings-sharelatex": @settings = - apis: - web: - url: "http://web.example.com" - user: "username" +describe('WebApiManager', function() { + beforeEach(function() { + this.project_id = "project-id-123"; + this.user_id = "user-id-123"; + this.user = {_id: this.user_id}; + this.callback = sinon.stub(); + return this.WebApiManager = SandboxedModule.require(modulePath, { requires: { + "request": (this.request = {}), + "settings-sharelatex": (this.settings = { + apis: { + web: { + url: "http://web.example.com", + user: "username", pass: "password" - "logger-sharelatex": @logger = { log: sinon.stub(), error: sinon.stub() } + } + } + }), + "logger-sharelatex": (this.logger = { log: sinon.stub(), error: sinon.stub() }) + } + });}); - describe "joinProject", -> - describe "successfully", -> - beforeEach -> - @response = { - project: { name: "Test project" } + return describe("joinProject", function() { + describe("successfully", function() { + beforeEach(function() { + this.response = { + project: { name: "Test project" }, privilegeLevel: "owner", isRestrictedUser: true - } - @request.post = sinon.stub().callsArgWith(1, null, {statusCode: 200}, @response) - @WebApiManager.joinProject @project_id, @user, @callback + }; + this.request.post = sinon.stub().callsArgWith(1, null, {statusCode: 200}, this.response); + return this.WebApiManager.joinProject(this.project_id, this.user, this.callback); + }); - it "should send a request to web to join the project", -> - @request.post + it("should send a request to web to join the project", function() { + return this.request.post .calledWith({ - url: "#{@settings.apis.web.url}/project/#{@project_id}/join" - qs: - user_id: @user_id - auth: - user: @settings.apis.web.user - pass: @settings.apis.web.pass + url: `${this.settings.apis.web.url}/project/${this.project_id}/join`, + qs: { + user_id: this.user_id + }, + auth: { + user: this.settings.apis.web.user, + pass: this.settings.apis.web.pass, sendImmediately: true - json: true - jar: false + }, + json: true, + jar: false, headers: {} }) - .should.equal true + .should.equal(true); + }); - it "should return the project, privilegeLevel, and restricted flag", -> - @callback - .calledWith(null, @response.project, @response.privilegeLevel, @response.isRestrictedUser) - .should.equal true + return it("should return the project, privilegeLevel, and restricted flag", function() { + return this.callback + .calledWith(null, this.response.project, this.response.privilegeLevel, this.response.isRestrictedUser) + .should.equal(true); + }); + }); - describe "with an error from web", -> - beforeEach -> - @request.post = sinon.stub().callsArgWith(1, null, {statusCode: 500}, null) - @WebApiManager.joinProject @project_id, @user_id, @callback + describe("with an error from web", function() { + beforeEach(function() { + this.request.post = sinon.stub().callsArgWith(1, null, {statusCode: 500}, null); + return this.WebApiManager.joinProject(this.project_id, this.user_id, this.callback); + }); - it "should call the callback with an error", -> - @callback + return it("should call the callback with an error", function() { + return this.callback .calledWith(sinon.match({message: "non-success status code from web: 500"})) - .should.equal true + .should.equal(true); + }); + }); - describe "with no data from web", -> - beforeEach -> - @request.post = sinon.stub().callsArgWith(1, null, {statusCode: 200}, null) - @WebApiManager.joinProject @project_id, @user_id, @callback + describe("with no data from web", function() { + beforeEach(function() { + this.request.post = sinon.stub().callsArgWith(1, null, {statusCode: 200}, null); + return this.WebApiManager.joinProject(this.project_id, this.user_id, this.callback); + }); - it "should call the callback with an error", -> - @callback + return it("should call the callback with an error", function() { + return this.callback .calledWith(sinon.match({message: "no data returned from joinProject request"})) - .should.equal true + .should.equal(true); + }); + }); - describe "when the project is over its rate limit", -> - beforeEach -> - @request.post = sinon.stub().callsArgWith(1, null, {statusCode: 429}, null) - @WebApiManager.joinProject @project_id, @user_id, @callback + return describe("when the project is over its rate limit", function() { + beforeEach(function() { + this.request.post = sinon.stub().callsArgWith(1, null, {statusCode: 429}, null); + return this.WebApiManager.joinProject(this.project_id, this.user_id, this.callback); + }); - it "should call the callback with a TooManyRequests error code", -> - @callback + return it("should call the callback with a TooManyRequests error code", function() { + return this.callback .calledWith(sinon.match({message: "rate-limit hit when joining project", code: "TooManyRequests"})) - .should.equal true + .should.equal(true); + }); + }); + }); +}); diff --git a/services/real-time/test/unit/coffee/WebsocketControllerTests.js b/services/real-time/test/unit/coffee/WebsocketControllerTests.js index c0047c49b7..92d64d7cd2 100644 --- a/services/real-time/test/unit/coffee/WebsocketControllerTests.js +++ b/services/real-time/test/unit/coffee/WebsocketControllerTests.js @@ -1,872 +1,1088 @@ -chai = require('chai') -should = chai.should() -sinon = require("sinon") -expect = chai.expect -modulePath = "../../../app/js/WebsocketController.js" -SandboxedModule = require('sandboxed-module') -tk = require "timekeeper" +/* + * decaffeinate suggestions: + * DS101: Remove unnecessary use of Array.from + * DS102: Remove unnecessary code created because of implicit returns + * Full docs: https://github.com/decaffeinate/decaffeinate/blob/master/docs/suggestions.md + */ +const chai = require('chai'); +const should = chai.should(); +const sinon = require("sinon"); +const { + expect +} = chai; +const modulePath = "../../../app/js/WebsocketController.js"; +const SandboxedModule = require('sandboxed-module'); +const tk = require("timekeeper"); -describe 'WebsocketController', -> - beforeEach -> - tk.freeze(new Date()) - @project_id = "project-id-123" - @user = { - _id: @user_id = "user-id-123" - first_name: "James" - last_name: "Allen" - email: "james@example.com" - signUpDate: new Date("2014-01-01") +describe('WebsocketController', function() { + beforeEach(function() { + tk.freeze(new Date()); + this.project_id = "project-id-123"; + this.user = { + _id: (this.user_id = "user-id-123"), + first_name: "James", + last_name: "Allen", + email: "james@example.com", + signUpDate: new Date("2014-01-01"), loginCount: 42 - } - @callback = sinon.stub() - @client = - disconnected: false - id: @client_id = "mock-client-id-123" - publicId: "other-id-#{Math.random()}" - ol_context: {} - join: sinon.stub() + }; + this.callback = sinon.stub(); + this.client = { + disconnected: false, + id: (this.client_id = "mock-client-id-123"), + publicId: `other-id-${Math.random()}`, + ol_context: {}, + join: sinon.stub(), leave: sinon.stub() - @WebsocketController = SandboxedModule.require modulePath, requires: - "./WebApiManager": @WebApiManager = {} - "./AuthorizationManager": @AuthorizationManager = {} - "./DocumentUpdaterManager": @DocumentUpdaterManager = {} - "./ConnectedUsersManager": @ConnectedUsersManager = {} - "./WebsocketLoadBalancer": @WebsocketLoadBalancer = {} - "logger-sharelatex": @logger = { log: sinon.stub(), error: sinon.stub(), warn: sinon.stub() } - "metrics-sharelatex": @metrics = - inc: sinon.stub() + }; + return this.WebsocketController = SandboxedModule.require(modulePath, { requires: { + "./WebApiManager": (this.WebApiManager = {}), + "./AuthorizationManager": (this.AuthorizationManager = {}), + "./DocumentUpdaterManager": (this.DocumentUpdaterManager = {}), + "./ConnectedUsersManager": (this.ConnectedUsersManager = {}), + "./WebsocketLoadBalancer": (this.WebsocketLoadBalancer = {}), + "logger-sharelatex": (this.logger = { log: sinon.stub(), error: sinon.stub(), warn: sinon.stub() }), + "metrics-sharelatex": (this.metrics = { + inc: sinon.stub(), set: sinon.stub() - "./RoomManager": @RoomManager = {} + }), + "./RoomManager": (this.RoomManager = {}) + } + });}); - afterEach -> - tk.reset() + afterEach(() => tk.reset()); - describe "joinProject", -> - describe "when authorised", -> - beforeEach -> - @client.id = "mock-client-id" - @project = { - name: "Test Project" + describe("joinProject", function() { + describe("when authorised", function() { + beforeEach(function() { + this.client.id = "mock-client-id"; + this.project = { + name: "Test Project", owner: { - _id: @owner_id = "mock-owner-id-123" + _id: (this.owner_id = "mock-owner-id-123") } - } - @privilegeLevel = "owner" - @ConnectedUsersManager.updateUserPosition = sinon.stub().callsArg(4) - @isRestrictedUser = true - @WebApiManager.joinProject = sinon.stub().callsArgWith(2, null, @project, @privilegeLevel, @isRestrictedUser) - @RoomManager.joinProject = sinon.stub().callsArg(2) - @WebsocketController.joinProject @client, @user, @project_id, @callback + }; + this.privilegeLevel = "owner"; + this.ConnectedUsersManager.updateUserPosition = sinon.stub().callsArg(4); + this.isRestrictedUser = true; + this.WebApiManager.joinProject = sinon.stub().callsArgWith(2, null, this.project, this.privilegeLevel, this.isRestrictedUser); + this.RoomManager.joinProject = sinon.stub().callsArg(2); + return this.WebsocketController.joinProject(this.client, this.user, this.project_id, this.callback); + }); - it "should load the project from web", -> - @WebApiManager.joinProject - .calledWith(@project_id, @user) - .should.equal true + it("should load the project from web", function() { + return this.WebApiManager.joinProject + .calledWith(this.project_id, this.user) + .should.equal(true); + }); - it "should join the project room", -> - @RoomManager.joinProject.calledWith(@client, @project_id).should.equal true + it("should join the project room", function() { + return this.RoomManager.joinProject.calledWith(this.client, this.project_id).should.equal(true); + }); - it "should set the privilege level on the client", -> - @client.ol_context["privilege_level"].should.equal @privilegeLevel - it "should set the user's id on the client", -> - @client.ol_context["user_id"].should.equal @user._id - it "should set the user's email on the client", -> - @client.ol_context["email"].should.equal @user.email - it "should set the user's first_name on the client", -> - @client.ol_context["first_name"].should.equal @user.first_name - it "should set the user's last_name on the client", -> - @client.ol_context["last_name"].should.equal @user.last_name - it "should set the user's sign up date on the client", -> - @client.ol_context["signup_date"].should.equal @user.signUpDate - it "should set the user's login_count on the client", -> - @client.ol_context["login_count"].should.equal @user.loginCount - it "should set the connected time on the client", -> - @client.ol_context["connected_time"].should.equal new Date() - it "should set the project_id on the client", -> - @client.ol_context["project_id"].should.equal @project_id - it "should set the project owner id on the client", -> - @client.ol_context["owner_id"].should.equal @owner_id - it "should set the is_restricted_user flag on the client", -> - @client.ol_context["is_restricted_user"].should.equal @isRestrictedUser - it "should call the callback with the project, privilegeLevel and protocolVersion", -> - @callback - .calledWith(null, @project, @privilegeLevel, @WebsocketController.PROTOCOL_VERSION) - .should.equal true + it("should set the privilege level on the client", function() { + return this.client.ol_context["privilege_level"].should.equal(this.privilegeLevel); + }); + it("should set the user's id on the client", function() { + return this.client.ol_context["user_id"].should.equal(this.user._id); + }); + it("should set the user's email on the client", function() { + return this.client.ol_context["email"].should.equal(this.user.email); + }); + it("should set the user's first_name on the client", function() { + return this.client.ol_context["first_name"].should.equal(this.user.first_name); + }); + it("should set the user's last_name on the client", function() { + return this.client.ol_context["last_name"].should.equal(this.user.last_name); + }); + it("should set the user's sign up date on the client", function() { + return this.client.ol_context["signup_date"].should.equal(this.user.signUpDate); + }); + it("should set the user's login_count on the client", function() { + return this.client.ol_context["login_count"].should.equal(this.user.loginCount); + }); + it("should set the connected time on the client", function() { + return this.client.ol_context["connected_time"].should.equal(new Date()); + }); + it("should set the project_id on the client", function() { + return this.client.ol_context["project_id"].should.equal(this.project_id); + }); + it("should set the project owner id on the client", function() { + return this.client.ol_context["owner_id"].should.equal(this.owner_id); + }); + it("should set the is_restricted_user flag on the client", function() { + return this.client.ol_context["is_restricted_user"].should.equal(this.isRestrictedUser); + }); + it("should call the callback with the project, privilegeLevel and protocolVersion", function() { + return this.callback + .calledWith(null, this.project, this.privilegeLevel, this.WebsocketController.PROTOCOL_VERSION) + .should.equal(true); + }); - it "should mark the user as connected in ConnectedUsersManager", -> - @ConnectedUsersManager.updateUserPosition - .calledWith(@project_id, @client.publicId, @user, null) - .should.equal true + it("should mark the user as connected in ConnectedUsersManager", function() { + return this.ConnectedUsersManager.updateUserPosition + .calledWith(this.project_id, this.client.publicId, this.user, null) + .should.equal(true); + }); - it "should increment the join-project metric", -> - @metrics.inc.calledWith("editor.join-project").should.equal true + return it("should increment the join-project metric", function() { + return this.metrics.inc.calledWith("editor.join-project").should.equal(true); + }); + }); - describe "when not authorized", -> - beforeEach -> - @WebApiManager.joinProject = sinon.stub().callsArgWith(2, null, null, null) - @WebsocketController.joinProject @client, @user, @project_id, @callback + describe("when not authorized", function() { + beforeEach(function() { + this.WebApiManager.joinProject = sinon.stub().callsArgWith(2, null, null, null); + return this.WebsocketController.joinProject(this.client, this.user, this.project_id, this.callback); + }); - it "should return an error", -> - @callback + it("should return an error", function() { + return this.callback .calledWith(sinon.match({message: "not authorized"})) - .should.equal true + .should.equal(true); + }); - it "should not log an error", -> - @logger.error.called.should.equal false + return it("should not log an error", function() { + return this.logger.error.called.should.equal(false); + }); + }); - describe "when the subscribe failed", -> - beforeEach -> - @client.id = "mock-client-id" - @project = { - name: "Test Project" + describe("when the subscribe failed", function() { + beforeEach(function() { + this.client.id = "mock-client-id"; + this.project = { + name: "Test Project", owner: { - _id: @owner_id = "mock-owner-id-123" + _id: (this.owner_id = "mock-owner-id-123") + } + }; + this.privilegeLevel = "owner"; + this.ConnectedUsersManager.updateUserPosition = sinon.stub().callsArg(4); + this.isRestrictedUser = true; + this.WebApiManager.joinProject = sinon.stub().callsArgWith(2, null, this.project, this.privilegeLevel, this.isRestrictedUser); + this.RoomManager.joinProject = sinon.stub().callsArgWith(2, new Error("subscribe failed")); + return this.WebsocketController.joinProject(this.client, this.user, this.project_id, this.callback); + }); + + return it("should return an error", function() { + this.callback + .calledWith(sinon.match({message: "subscribe failed"})) + .should.equal(true); + return this.callback.args[0][0].message.should.equal("subscribe failed"); + }); + }); + + describe("when the client has disconnected", function() { + beforeEach(function() { + this.client.disconnected = true; + this.WebApiManager.joinProject = sinon.stub().callsArg(2); + return this.WebsocketController.joinProject(this.client, this.user, this.project_id, this.callback); + }); + + it("should not call WebApiManager.joinProject", function() { + return expect(this.WebApiManager.joinProject.called).to.equal(false); + }); + + it("should call the callback with no details", function() { + return expect(this.callback.args[0]).to.deep.equal([]); + }); + + return it("should increment the editor.join-project.disconnected metric with a status", function() { + return expect(this.metrics.inc.calledWith('editor.join-project.disconnected', 1, {status: 'immediately'})).to.equal(true); + }); + }); + + return describe("when the client disconnects while WebApiManager.joinProject is running", function() { + beforeEach(function() { + this.WebApiManager.joinProject = (project, user, cb) => { + this.client.disconnected = true; + return cb(null, this.project, this.privilegeLevel, this.isRestrictedUser); + }; + + return this.WebsocketController.joinProject(this.client, this.user, this.project_id, this.callback); + }); + + it("should call the callback with no details", function() { + return expect(this.callback.args[0]).to.deep.equal([]); + }); + + return it("should increment the editor.join-project.disconnected metric with a status", function() { + return expect(this.metrics.inc.calledWith('editor.join-project.disconnected', 1, {status: 'after-web-api-call'})).to.equal(true); + }); + }); + }); + + describe("leaveProject", function() { + beforeEach(function() { + this.DocumentUpdaterManager.flushProjectToMongoAndDelete = sinon.stub().callsArg(1); + this.ConnectedUsersManager.markUserAsDisconnected = sinon.stub().callsArg(2); + this.WebsocketLoadBalancer.emitToRoom = sinon.stub(); + this.RoomManager.leaveProjectAndDocs = sinon.stub(); + this.clientsInRoom = []; + this.io = { + sockets: { + clients: room_id => { + if (room_id !== this.project_id) { + throw "expected room_id to be project_id"; + } + return this.clientsInRoom; } } - @privilegeLevel = "owner" - @ConnectedUsersManager.updateUserPosition = sinon.stub().callsArg(4) - @isRestrictedUser = true - @WebApiManager.joinProject = sinon.stub().callsArgWith(2, null, @project, @privilegeLevel, @isRestrictedUser) - @RoomManager.joinProject = sinon.stub().callsArgWith(2, new Error("subscribe failed")) - @WebsocketController.joinProject @client, @user, @project_id, @callback - - it "should return an error", -> - @callback - .calledWith(sinon.match({message: "subscribe failed"})) - .should.equal true - @callback.args[0][0].message.should.equal "subscribe failed" - - describe "when the client has disconnected", -> - beforeEach -> - @client.disconnected = true - @WebApiManager.joinProject = sinon.stub().callsArg(2) - @WebsocketController.joinProject @client, @user, @project_id, @callback - - it "should not call WebApiManager.joinProject", -> - expect(@WebApiManager.joinProject.called).to.equal(false) - - it "should call the callback with no details", -> - expect(@callback.args[0]).to.deep.equal [] - - it "should increment the editor.join-project.disconnected metric with a status", -> - expect(@metrics.inc.calledWith('editor.join-project.disconnected', 1, {status: 'immediately'})).to.equal(true) - - describe "when the client disconnects while WebApiManager.joinProject is running", -> - beforeEach -> - @WebApiManager.joinProject = (project, user, cb) => - @client.disconnected = true - cb(null, @project, @privilegeLevel, @isRestrictedUser) - - @WebsocketController.joinProject @client, @user, @project_id, @callback - - it "should call the callback with no details", -> - expect(@callback.args[0]).to.deep.equal [] - - it "should increment the editor.join-project.disconnected metric with a status", -> - expect(@metrics.inc.calledWith('editor.join-project.disconnected', 1, {status: 'after-web-api-call'})).to.equal(true) - - describe "leaveProject", -> - beforeEach -> - @DocumentUpdaterManager.flushProjectToMongoAndDelete = sinon.stub().callsArg(1) - @ConnectedUsersManager.markUserAsDisconnected = sinon.stub().callsArg(2) - @WebsocketLoadBalancer.emitToRoom = sinon.stub() - @RoomManager.leaveProjectAndDocs = sinon.stub() - @clientsInRoom = [] - @io = - sockets: - clients: (room_id) => - if room_id != @project_id - throw "expected room_id to be project_id" - return @clientsInRoom - @client.ol_context.project_id = @project_id - @client.ol_context.user_id = @user_id - @WebsocketController.FLUSH_IF_EMPTY_DELAY = 0 - tk.reset() # Allow setTimeout to work. - - describe "when the client did not joined a project yet", -> - beforeEach (done) -> - @client.ol_context = {} - @WebsocketController.leaveProject @io, @client, done - - it "should bail out when calling leaveProject", () -> - @WebsocketLoadBalancer.emitToRoom.called.should.equal false - @RoomManager.leaveProjectAndDocs.called.should.equal false - @ConnectedUsersManager.markUserAsDisconnected.called.should.equal false - - it "should not inc any metric", () -> - @metrics.inc.called.should.equal false - - describe "when the project is empty", -> - beforeEach (done) -> - @clientsInRoom = [] - @WebsocketController.leaveProject @io, @client, done - - it "should end clientTracking.clientDisconnected to the project room", -> - @WebsocketLoadBalancer.emitToRoom - .calledWith(@project_id, "clientTracking.clientDisconnected", @client.publicId) - .should.equal true - - it "should mark the user as disconnected", -> - @ConnectedUsersManager.markUserAsDisconnected - .calledWith(@project_id, @client.publicId) - .should.equal true - - it "should flush the project in the document updater", -> - @DocumentUpdaterManager.flushProjectToMongoAndDelete - .calledWith(@project_id) - .should.equal true - - it "should increment the leave-project metric", -> - @metrics.inc.calledWith("editor.leave-project").should.equal true - - it "should track the disconnection in RoomManager", -> - @RoomManager.leaveProjectAndDocs - .calledWith(@client) - .should.equal true - - describe "when the project is not empty", -> - beforeEach -> - @clientsInRoom = ["mock-remaining-client"] - @WebsocketController.leaveProject @io, @client - - it "should not flush the project in the document updater", -> - @DocumentUpdaterManager.flushProjectToMongoAndDelete - .called.should.equal false - - describe "when client has not authenticated", -> - beforeEach (done) -> - @client.ol_context.user_id = null - @client.ol_context.project_id = null - @WebsocketController.leaveProject @io, @client, done - - it "should not end clientTracking.clientDisconnected to the project room", -> - @WebsocketLoadBalancer.emitToRoom - .calledWith(@project_id, "clientTracking.clientDisconnected", @client.publicId) - .should.equal false - - it "should not mark the user as disconnected", -> - @ConnectedUsersManager.markUserAsDisconnected - .calledWith(@project_id, @client.publicId) - .should.equal false - - it "should not flush the project in the document updater", -> - @DocumentUpdaterManager.flushProjectToMongoAndDelete - .calledWith(@project_id) - .should.equal false - - it "should not increment the leave-project metric", -> - @metrics.inc.calledWith("editor.leave-project").should.equal false - - describe "when client has not joined a project", -> - beforeEach (done) -> - @client.ol_context.user_id = @user_id - @client.ol_context.project_id = null - @WebsocketController.leaveProject @io, @client, done - - it "should not end clientTracking.clientDisconnected to the project room", -> - @WebsocketLoadBalancer.emitToRoom - .calledWith(@project_id, "clientTracking.clientDisconnected", @client.publicId) - .should.equal false - - it "should not mark the user as disconnected", -> - @ConnectedUsersManager.markUserAsDisconnected - .calledWith(@project_id, @client.publicId) - .should.equal false - - it "should not flush the project in the document updater", -> - @DocumentUpdaterManager.flushProjectToMongoAndDelete - .calledWith(@project_id) - .should.equal false - - it "should not increment the leave-project metric", -> - @metrics.inc.calledWith("editor.leave-project").should.equal false - - describe "joinDoc", -> - beforeEach -> - @doc_id = "doc-id-123" - @doc_lines = ["doc", "lines"] - @version = 42 - @ops = ["mock", "ops"] - @ranges = { "mock": "ranges" } - @options = {} - - @client.ol_context.project_id = @project_id - @client.ol_context.is_restricted_user = false - @AuthorizationManager.addAccessToDoc = sinon.stub() - @AuthorizationManager.assertClientCanViewProject = sinon.stub().callsArgWith(1, null) - @DocumentUpdaterManager.getDocument = sinon.stub().callsArgWith(3, null, @doc_lines, @version, @ranges, @ops) - @RoomManager.joinDoc = sinon.stub().callsArg(2) - - describe "works", -> - beforeEach -> - @WebsocketController.joinDoc @client, @doc_id, -1, @options, @callback - - it "should check that the client is authorized to view the project", -> - @AuthorizationManager.assertClientCanViewProject - .calledWith(@client) - .should.equal true - - it "should get the document from the DocumentUpdaterManager with fromVersion", -> - @DocumentUpdaterManager.getDocument - .calledWith(@project_id, @doc_id, -1) - .should.equal true - - it "should add permissions for the client to access the doc", -> - @AuthorizationManager.addAccessToDoc - .calledWith(@client, @doc_id) - .should.equal true - - it "should join the client to room for the doc_id", -> - @RoomManager.joinDoc - .calledWith(@client, @doc_id) - .should.equal true - - it "should call the callback with the lines, version, ranges and ops", -> - @callback - .calledWith(null, @doc_lines, @version, @ops, @ranges) - .should.equal true - - it "should increment the join-doc metric", -> - @metrics.inc.calledWith("editor.join-doc").should.equal true - - describe "with a fromVersion", -> - beforeEach -> - @fromVersion = 40 - @WebsocketController.joinDoc @client, @doc_id, @fromVersion, @options, @callback - - it "should get the document from the DocumentUpdaterManager with fromVersion", -> - @DocumentUpdaterManager.getDocument - .calledWith(@project_id, @doc_id, @fromVersion) - .should.equal true - - describe "with doclines that need escaping", -> - beforeEach -> - @doc_lines.push ["räksmörgås"] - @WebsocketController.joinDoc @client, @doc_id, -1, @options, @callback - - it "should call the callback with the escaped lines", -> - escaped_lines = @callback.args[0][1] - escaped_word = escaped_lines.pop() - escaped_word.should.equal 'räksmörgÃ¥s' - # Check that unescaping works - decodeURIComponent(escape(escaped_word)).should.equal "räksmörgås" - - describe "with comments that need encoding", -> - beforeEach -> - @ranges.comments = [{ op: { c: "räksmörgås" } }] - @WebsocketController.joinDoc @client, @doc_id, -1, { encodeRanges: true }, @callback - - it "should call the callback with the encoded comment", -> - encoded_comments = @callback.args[0][4] - encoded_comment = encoded_comments.comments.pop() - encoded_comment_text = encoded_comment.op.c - encoded_comment_text.should.equal 'räksmörgÃ¥s' - - describe "with changes that need encoding", -> - it "should call the callback with the encoded insert change", -> - @ranges.changes = [{ op: { i: "räksmörgås" } }] - @WebsocketController.joinDoc @client, @doc_id, -1, { encodeRanges: true }, @callback - - encoded_changes = @callback.args[0][4] - encoded_change = encoded_changes.changes.pop() - encoded_change_text = encoded_change.op.i - encoded_change_text.should.equal 'räksmörgÃ¥s' - - it "should call the callback with the encoded delete change", -> - @ranges.changes = [{ op: { d: "räksmörgås" } }] - @WebsocketController.joinDoc @client, @doc_id, -1, { encodeRanges: true }, @callback - - encoded_changes = @callback.args[0][4] - encoded_change = encoded_changes.changes.pop() - encoded_change_text = encoded_change.op.d - encoded_change_text.should.equal 'räksmörgÃ¥s' - - describe "when not authorized", -> - beforeEach -> - @AuthorizationManager.assertClientCanViewProject = sinon.stub().callsArgWith(1, @err = new Error("not authorized")) - @WebsocketController.joinDoc @client, @doc_id, -1, @options, @callback - - it "should call the callback with an error", -> - @callback.calledWith(sinon.match({message: "not authorized"})).should.equal true - - it "should not call the DocumentUpdaterManager", -> - @DocumentUpdaterManager.getDocument.called.should.equal false - - describe "with a restricted client", -> - beforeEach -> - @ranges.comments = [{op: {a: 1}}, {op: {a: 2}}] - @client.ol_context.is_restricted_user = true - @WebsocketController.joinDoc @client, @doc_id, -1, @options, @callback - - it "should overwrite ranges.comments with an empty list", -> - ranges = @callback.args[0][4] - expect(ranges.comments).to.deep.equal [] - - describe "when the client has disconnected", -> - beforeEach -> - @client.disconnected = true - @WebsocketController.joinDoc @client, @doc_id, -1, @options, @callback - - it "should call the callback with no details", -> - expect(@callback.args[0]).to.deep.equal([]) - - it "should increment the editor.join-doc.disconnected metric with a status", -> - expect(@metrics.inc.calledWith('editor.join-doc.disconnected', 1, {status: 'immediately'})).to.equal(true) - - it "should not get the document", -> - expect(@DocumentUpdaterManager.getDocument.called).to.equal(false) - - describe "when the client disconnects while RoomManager.joinDoc is running", -> - beforeEach -> - @RoomManager.joinDoc = (client, doc_id, cb) => - @client.disconnected = true - cb() - - @WebsocketController.joinDoc @client, @doc_id, -1, @options, @callback - - it "should call the callback with no details", -> - expect(@callback.args[0]).to.deep.equal([]) - - it "should increment the editor.join-doc.disconnected metric with a status", -> - expect(@metrics.inc.calledWith('editor.join-doc.disconnected', 1, {status: 'after-joining-room'})).to.equal(true) - - it "should not get the document", -> - expect(@DocumentUpdaterManager.getDocument.called).to.equal(false) - - describe "when the client disconnects while DocumentUpdaterManager.getDocument is running", -> - beforeEach -> - @DocumentUpdaterManager.getDocument = (project_id, doc_id, fromVersion, callback) => - @client.disconnected = true - callback(null, @doc_lines, @version, @ranges, @ops) - - @WebsocketController.joinDoc @client, @doc_id, -1, @options, @callback - - it "should call the callback with no details", -> - expect(@callback.args[0]).to.deep.equal [] - - it "should increment the editor.join-doc.disconnected metric with a status", -> - expect(@metrics.inc.calledWith('editor.join-doc.disconnected', 1, {status: 'after-doc-updater-call'})).to.equal(true) - - describe "leaveDoc", -> - beforeEach -> - @doc_id = "doc-id-123" - @client.ol_context.project_id = @project_id - @RoomManager.leaveDoc = sinon.stub() - @WebsocketController.leaveDoc @client, @doc_id, @callback - - it "should remove the client from the doc_id room", -> - @RoomManager.leaveDoc - .calledWith(@client, @doc_id).should.equal true - - it "should call the callback", -> - @callback.called.should.equal true - - it "should increment the leave-doc metric", -> - @metrics.inc.calledWith("editor.leave-doc").should.equal true - - describe "getConnectedUsers", -> - beforeEach -> - @client.ol_context.project_id = @project_id - @users = ["mock", "users"] - @WebsocketLoadBalancer.emitToRoom = sinon.stub() - @ConnectedUsersManager.getConnectedUsers = sinon.stub().callsArgWith(1, null, @users) - - describe "when authorized", -> - beforeEach (done) -> - @AuthorizationManager.assertClientCanViewProject = sinon.stub().callsArgWith(1, null) - @WebsocketController.getConnectedUsers @client, (args...) => - @callback(args...) - done() - - it "should check that the client is authorized to view the project", -> - @AuthorizationManager.assertClientCanViewProject - .calledWith(@client) - .should.equal true - - it "should broadcast a request to update the client list", -> - @WebsocketLoadBalancer.emitToRoom - .calledWith(@project_id, "clientTracking.refresh") - .should.equal true - - it "should get the connected users for the project", -> - @ConnectedUsersManager.getConnectedUsers - .calledWith(@project_id) - .should.equal true - - it "should return the users", -> - @callback.calledWith(null, @users).should.equal true - - it "should increment the get-connected-users metric", -> - @metrics.inc.calledWith("editor.get-connected-users").should.equal true - - describe "when not authorized", -> - beforeEach -> - @AuthorizationManager.assertClientCanViewProject = sinon.stub().callsArgWith(1, @err = new Error("not authorized")) - @WebsocketController.getConnectedUsers @client, @callback - - it "should not get the connected users for the project", -> - @ConnectedUsersManager.getConnectedUsers + }; + this.client.ol_context.project_id = this.project_id; + this.client.ol_context.user_id = this.user_id; + this.WebsocketController.FLUSH_IF_EMPTY_DELAY = 0; + return tk.reset(); + }); // Allow setTimeout to work. + + describe("when the client did not joined a project yet", function() { + beforeEach(function(done) { + this.client.ol_context = {}; + return this.WebsocketController.leaveProject(this.io, this.client, done); + }); + + it("should bail out when calling leaveProject", function() { + this.WebsocketLoadBalancer.emitToRoom.called.should.equal(false); + this.RoomManager.leaveProjectAndDocs.called.should.equal(false); + return this.ConnectedUsersManager.markUserAsDisconnected.called.should.equal(false); + }); + + return it("should not inc any metric", function() { + return this.metrics.inc.called.should.equal(false); + }); + }); + + describe("when the project is empty", function() { + beforeEach(function(done) { + this.clientsInRoom = []; + return this.WebsocketController.leaveProject(this.io, this.client, done); + }); + + it("should end clientTracking.clientDisconnected to the project room", function() { + return this.WebsocketLoadBalancer.emitToRoom + .calledWith(this.project_id, "clientTracking.clientDisconnected", this.client.publicId) + .should.equal(true); + }); + + it("should mark the user as disconnected", function() { + return this.ConnectedUsersManager.markUserAsDisconnected + .calledWith(this.project_id, this.client.publicId) + .should.equal(true); + }); + + it("should flush the project in the document updater", function() { + return this.DocumentUpdaterManager.flushProjectToMongoAndDelete + .calledWith(this.project_id) + .should.equal(true); + }); + + it("should increment the leave-project metric", function() { + return this.metrics.inc.calledWith("editor.leave-project").should.equal(true); + }); + + return it("should track the disconnection in RoomManager", function() { + return this.RoomManager.leaveProjectAndDocs + .calledWith(this.client) + .should.equal(true); + }); + }); + + describe("when the project is not empty", function() { + beforeEach(function() { + this.clientsInRoom = ["mock-remaining-client"]; + return this.WebsocketController.leaveProject(this.io, this.client); + }); + + return it("should not flush the project in the document updater", function() { + return this.DocumentUpdaterManager.flushProjectToMongoAndDelete + .called.should.equal(false); + }); + }); + + describe("when client has not authenticated", function() { + beforeEach(function(done) { + this.client.ol_context.user_id = null; + this.client.ol_context.project_id = null; + return this.WebsocketController.leaveProject(this.io, this.client, done); + }); + + it("should not end clientTracking.clientDisconnected to the project room", function() { + return this.WebsocketLoadBalancer.emitToRoom + .calledWith(this.project_id, "clientTracking.clientDisconnected", this.client.publicId) + .should.equal(false); + }); + + it("should not mark the user as disconnected", function() { + return this.ConnectedUsersManager.markUserAsDisconnected + .calledWith(this.project_id, this.client.publicId) + .should.equal(false); + }); + + it("should not flush the project in the document updater", function() { + return this.DocumentUpdaterManager.flushProjectToMongoAndDelete + .calledWith(this.project_id) + .should.equal(false); + }); + + return it("should not increment the leave-project metric", function() { + return this.metrics.inc.calledWith("editor.leave-project").should.equal(false); + }); + }); + + return describe("when client has not joined a project", function() { + beforeEach(function(done) { + this.client.ol_context.user_id = this.user_id; + this.client.ol_context.project_id = null; + return this.WebsocketController.leaveProject(this.io, this.client, done); + }); + + it("should not end clientTracking.clientDisconnected to the project room", function() { + return this.WebsocketLoadBalancer.emitToRoom + .calledWith(this.project_id, "clientTracking.clientDisconnected", this.client.publicId) + .should.equal(false); + }); + + it("should not mark the user as disconnected", function() { + return this.ConnectedUsersManager.markUserAsDisconnected + .calledWith(this.project_id, this.client.publicId) + .should.equal(false); + }); + + it("should not flush the project in the document updater", function() { + return this.DocumentUpdaterManager.flushProjectToMongoAndDelete + .calledWith(this.project_id) + .should.equal(false); + }); + + return it("should not increment the leave-project metric", function() { + return this.metrics.inc.calledWith("editor.leave-project").should.equal(false); + }); + }); + }); + + describe("joinDoc", function() { + beforeEach(function() { + this.doc_id = "doc-id-123"; + this.doc_lines = ["doc", "lines"]; + this.version = 42; + this.ops = ["mock", "ops"]; + this.ranges = { "mock": "ranges" }; + this.options = {}; + + this.client.ol_context.project_id = this.project_id; + this.client.ol_context.is_restricted_user = false; + this.AuthorizationManager.addAccessToDoc = sinon.stub(); + this.AuthorizationManager.assertClientCanViewProject = sinon.stub().callsArgWith(1, null); + this.DocumentUpdaterManager.getDocument = sinon.stub().callsArgWith(3, null, this.doc_lines, this.version, this.ranges, this.ops); + return this.RoomManager.joinDoc = sinon.stub().callsArg(2); + }); + + describe("works", function() { + beforeEach(function() { + return this.WebsocketController.joinDoc(this.client, this.doc_id, -1, this.options, this.callback); + }); + + it("should check that the client is authorized to view the project", function() { + return this.AuthorizationManager.assertClientCanViewProject + .calledWith(this.client) + .should.equal(true); + }); + + it("should get the document from the DocumentUpdaterManager with fromVersion", function() { + return this.DocumentUpdaterManager.getDocument + .calledWith(this.project_id, this.doc_id, -1) + .should.equal(true); + }); + + it("should add permissions for the client to access the doc", function() { + return this.AuthorizationManager.addAccessToDoc + .calledWith(this.client, this.doc_id) + .should.equal(true); + }); + + it("should join the client to room for the doc_id", function() { + return this.RoomManager.joinDoc + .calledWith(this.client, this.doc_id) + .should.equal(true); + }); + + it("should call the callback with the lines, version, ranges and ops", function() { + return this.callback + .calledWith(null, this.doc_lines, this.version, this.ops, this.ranges) + .should.equal(true); + }); + + return it("should increment the join-doc metric", function() { + return this.metrics.inc.calledWith("editor.join-doc").should.equal(true); + }); + }); + + describe("with a fromVersion", function() { + beforeEach(function() { + this.fromVersion = 40; + return this.WebsocketController.joinDoc(this.client, this.doc_id, this.fromVersion, this.options, this.callback); + }); + + return it("should get the document from the DocumentUpdaterManager with fromVersion", function() { + return this.DocumentUpdaterManager.getDocument + .calledWith(this.project_id, this.doc_id, this.fromVersion) + .should.equal(true); + }); + }); + + describe("with doclines that need escaping", function() { + beforeEach(function() { + this.doc_lines.push(["räksmörgås"]); + return this.WebsocketController.joinDoc(this.client, this.doc_id, -1, this.options, this.callback); + }); + + return it("should call the callback with the escaped lines", function() { + const escaped_lines = this.callback.args[0][1]; + const escaped_word = escaped_lines.pop(); + escaped_word.should.equal('räksmörgÃ¥s'); + // Check that unescaping works + return decodeURIComponent(escape(escaped_word)).should.equal("räksmörgås"); + }); + }); + + describe("with comments that need encoding", function() { + beforeEach(function() { + this.ranges.comments = [{ op: { c: "räksmörgås" } }]; + return this.WebsocketController.joinDoc(this.client, this.doc_id, -1, { encodeRanges: true }, this.callback); + }); + + return it("should call the callback with the encoded comment", function() { + const encoded_comments = this.callback.args[0][4]; + const encoded_comment = encoded_comments.comments.pop(); + const encoded_comment_text = encoded_comment.op.c; + return encoded_comment_text.should.equal('räksmörgÃ¥s'); + }); + }); + + describe("with changes that need encoding", function() { + it("should call the callback with the encoded insert change", function() { + this.ranges.changes = [{ op: { i: "räksmörgås" } }]; + this.WebsocketController.joinDoc(this.client, this.doc_id, -1, { encodeRanges: true }, this.callback); + + const encoded_changes = this.callback.args[0][4]; + const encoded_change = encoded_changes.changes.pop(); + const encoded_change_text = encoded_change.op.i; + return encoded_change_text.should.equal('räksmörgÃ¥s'); + }); + + return it("should call the callback with the encoded delete change", function() { + this.ranges.changes = [{ op: { d: "räksmörgås" } }]; + this.WebsocketController.joinDoc(this.client, this.doc_id, -1, { encodeRanges: true }, this.callback); + + const encoded_changes = this.callback.args[0][4]; + const encoded_change = encoded_changes.changes.pop(); + const encoded_change_text = encoded_change.op.d; + return encoded_change_text.should.equal('räksmörgÃ¥s'); + }); + }); + + describe("when not authorized", function() { + beforeEach(function() { + this.AuthorizationManager.assertClientCanViewProject = sinon.stub().callsArgWith(1, (this.err = new Error("not authorized"))); + return this.WebsocketController.joinDoc(this.client, this.doc_id, -1, this.options, this.callback); + }); + + it("should call the callback with an error", function() { + return this.callback.calledWith(sinon.match({message: "not authorized"})).should.equal(true); + }); + + return it("should not call the DocumentUpdaterManager", function() { + return this.DocumentUpdaterManager.getDocument.called.should.equal(false); + }); + }); + + describe("with a restricted client", function() { + beforeEach(function() { + this.ranges.comments = [{op: {a: 1}}, {op: {a: 2}}]; + this.client.ol_context.is_restricted_user = true; + return this.WebsocketController.joinDoc(this.client, this.doc_id, -1, this.options, this.callback); + }); + + return it("should overwrite ranges.comments with an empty list", function() { + const ranges = this.callback.args[0][4]; + return expect(ranges.comments).to.deep.equal([]); + }); + }); + + describe("when the client has disconnected", function() { + beforeEach(function() { + this.client.disconnected = true; + return this.WebsocketController.joinDoc(this.client, this.doc_id, -1, this.options, this.callback); + }); + + it("should call the callback with no details", function() { + return expect(this.callback.args[0]).to.deep.equal([]); + }); + + it("should increment the editor.join-doc.disconnected metric with a status", function() { + return expect(this.metrics.inc.calledWith('editor.join-doc.disconnected', 1, {status: 'immediately'})).to.equal(true); + }); + + return it("should not get the document", function() { + return expect(this.DocumentUpdaterManager.getDocument.called).to.equal(false); + }); + }); + + describe("when the client disconnects while RoomManager.joinDoc is running", function() { + beforeEach(function() { + this.RoomManager.joinDoc = (client, doc_id, cb) => { + this.client.disconnected = true; + return cb(); + }; + + return this.WebsocketController.joinDoc(this.client, this.doc_id, -1, this.options, this.callback); + }); + + it("should call the callback with no details", function() { + return expect(this.callback.args[0]).to.deep.equal([]); + }); + + it("should increment the editor.join-doc.disconnected metric with a status", function() { + return expect(this.metrics.inc.calledWith('editor.join-doc.disconnected', 1, {status: 'after-joining-room'})).to.equal(true); + }); + + return it("should not get the document", function() { + return expect(this.DocumentUpdaterManager.getDocument.called).to.equal(false); + }); + }); + + return describe("when the client disconnects while DocumentUpdaterManager.getDocument is running", function() { + beforeEach(function() { + this.DocumentUpdaterManager.getDocument = (project_id, doc_id, fromVersion, callback) => { + this.client.disconnected = true; + return callback(null, this.doc_lines, this.version, this.ranges, this.ops); + }; + + return this.WebsocketController.joinDoc(this.client, this.doc_id, -1, this.options, this.callback); + }); + + it("should call the callback with no details", function() { + return expect(this.callback.args[0]).to.deep.equal([]); + }); + + return it("should increment the editor.join-doc.disconnected metric with a status", function() { + return expect(this.metrics.inc.calledWith('editor.join-doc.disconnected', 1, {status: 'after-doc-updater-call'})).to.equal(true); + }); + }); + }); + + describe("leaveDoc", function() { + beforeEach(function() { + this.doc_id = "doc-id-123"; + this.client.ol_context.project_id = this.project_id; + this.RoomManager.leaveDoc = sinon.stub(); + return this.WebsocketController.leaveDoc(this.client, this.doc_id, this.callback); + }); + + it("should remove the client from the doc_id room", function() { + return this.RoomManager.leaveDoc + .calledWith(this.client, this.doc_id).should.equal(true); + }); + + it("should call the callback", function() { + return this.callback.called.should.equal(true); + }); + + return it("should increment the leave-doc metric", function() { + return this.metrics.inc.calledWith("editor.leave-doc").should.equal(true); + }); + }); + + describe("getConnectedUsers", function() { + beforeEach(function() { + this.client.ol_context.project_id = this.project_id; + this.users = ["mock", "users"]; + this.WebsocketLoadBalancer.emitToRoom = sinon.stub(); + return this.ConnectedUsersManager.getConnectedUsers = sinon.stub().callsArgWith(1, null, this.users); + }); + + describe("when authorized", function() { + beforeEach(function(done) { + this.AuthorizationManager.assertClientCanViewProject = sinon.stub().callsArgWith(1, null); + return this.WebsocketController.getConnectedUsers(this.client, (...args) => { + this.callback(...Array.from(args || [])); + return done(); + }); + }); + + it("should check that the client is authorized to view the project", function() { + return this.AuthorizationManager.assertClientCanViewProject + .calledWith(this.client) + .should.equal(true); + }); + + it("should broadcast a request to update the client list", function() { + return this.WebsocketLoadBalancer.emitToRoom + .calledWith(this.project_id, "clientTracking.refresh") + .should.equal(true); + }); + + it("should get the connected users for the project", function() { + return this.ConnectedUsersManager.getConnectedUsers + .calledWith(this.project_id) + .should.equal(true); + }); + + it("should return the users", function() { + return this.callback.calledWith(null, this.users).should.equal(true); + }); + + return it("should increment the get-connected-users metric", function() { + return this.metrics.inc.calledWith("editor.get-connected-users").should.equal(true); + }); + }); + + describe("when not authorized", function() { + beforeEach(function() { + this.AuthorizationManager.assertClientCanViewProject = sinon.stub().callsArgWith(1, (this.err = new Error("not authorized"))); + return this.WebsocketController.getConnectedUsers(this.client, this.callback); + }); + + it("should not get the connected users for the project", function() { + return this.ConnectedUsersManager.getConnectedUsers .called - .should.equal false + .should.equal(false); + }); - it "should return an error", -> - @callback.calledWith(@err).should.equal true + return it("should return an error", function() { + return this.callback.calledWith(this.err).should.equal(true); + }); + }); - describe "when restricted user", -> - beforeEach -> - @client.ol_context.is_restricted_user = true - @AuthorizationManager.assertClientCanViewProject = sinon.stub().callsArgWith(1, null) - @WebsocketController.getConnectedUsers @client, @callback + describe("when restricted user", function() { + beforeEach(function() { + this.client.ol_context.is_restricted_user = true; + this.AuthorizationManager.assertClientCanViewProject = sinon.stub().callsArgWith(1, null); + return this.WebsocketController.getConnectedUsers(this.client, this.callback); + }); - it "should return an empty array of users", -> - @callback.calledWith(null, []).should.equal true + it("should return an empty array of users", function() { + return this.callback.calledWith(null, []).should.equal(true); + }); - it "should not get the connected users for the project", -> - @ConnectedUsersManager.getConnectedUsers + return it("should not get the connected users for the project", function() { + return this.ConnectedUsersManager.getConnectedUsers .called - .should.equal false + .should.equal(false); + }); + }); - describe "when the client has disconnected", -> - beforeEach -> - @client.disconnected = true - @AuthorizationManager.assertClientCanViewProject = sinon.stub() - @WebsocketController.getConnectedUsers @client, @callback + return describe("when the client has disconnected", function() { + beforeEach(function() { + this.client.disconnected = true; + this.AuthorizationManager.assertClientCanViewProject = sinon.stub(); + return this.WebsocketController.getConnectedUsers(this.client, this.callback); + }); - it "should call the callback with no details", -> - expect(@callback.args[0]).to.deep.equal([]) + it("should call the callback with no details", function() { + return expect(this.callback.args[0]).to.deep.equal([]); + }); - it "should not check permissions", -> - expect(@AuthorizationManager.assertClientCanViewProject.called).to.equal(false) + return it("should not check permissions", function() { + return expect(this.AuthorizationManager.assertClientCanViewProject.called).to.equal(false); + }); + }); + }); - describe "updateClientPosition", -> - beforeEach -> - @WebsocketLoadBalancer.emitToRoom = sinon.stub() - @ConnectedUsersManager.updateUserPosition = sinon.stub().callsArgWith(4) - @AuthorizationManager.assertClientCanViewProjectAndDoc = sinon.stub().callsArgWith(2, null) - @update = { - doc_id: @doc_id = "doc-id-123" - row: @row = 42 - column: @column = 37 - } + describe("updateClientPosition", function() { + beforeEach(function() { + this.WebsocketLoadBalancer.emitToRoom = sinon.stub(); + this.ConnectedUsersManager.updateUserPosition = sinon.stub().callsArgWith(4); + this.AuthorizationManager.assertClientCanViewProjectAndDoc = sinon.stub().callsArgWith(2, null); + return this.update = { + doc_id: (this.doc_id = "doc-id-123"), + row: (this.row = 42), + column: (this.column = 37) + };}); - describe "with a logged in user", -> - beforeEach -> - @client.ol_context = { - project_id: @project_id - first_name: @first_name = "Douglas" - last_name: @last_name = "Adams" - email: @email = "joe@example.com" - user_id: @user_id = "user-id-123" - } - @WebsocketController.updateClientPosition @client, @update + describe("with a logged in user", function() { + beforeEach(function() { + this.client.ol_context = { + project_id: this.project_id, + first_name: (this.first_name = "Douglas"), + last_name: (this.last_name = "Adams"), + email: (this.email = "joe@example.com"), + user_id: (this.user_id = "user-id-123") + }; + this.WebsocketController.updateClientPosition(this.client, this.update); - @populatedCursorData = - doc_id: @doc_id, - id: @client.publicId - name: "#{@first_name} #{@last_name}" - row: @row - column: @column - email: @email - user_id: @user_id + return this.populatedCursorData = { + doc_id: this.doc_id, + id: this.client.publicId, + name: `${this.first_name} ${this.last_name}`, + row: this.row, + column: this.column, + email: this.email, + user_id: this.user_id + }; + }); - it "should send the update to the project room with the user's name", -> - @WebsocketLoadBalancer.emitToRoom.calledWith(@project_id, "clientTracking.clientUpdated", @populatedCursorData).should.equal true + it("should send the update to the project room with the user's name", function() { + return this.WebsocketLoadBalancer.emitToRoom.calledWith(this.project_id, "clientTracking.clientUpdated", this.populatedCursorData).should.equal(true); + }); - it "should send the cursor data to the connected user manager", (done)-> - @ConnectedUsersManager.updateUserPosition.calledWith(@project_id, @client.publicId, { - _id: @user_id, - email: @email, - first_name: @first_name, - last_name: @last_name + it("should send the cursor data to the connected user manager", function(done){ + this.ConnectedUsersManager.updateUserPosition.calledWith(this.project_id, this.client.publicId, { + _id: this.user_id, + email: this.email, + first_name: this.first_name, + last_name: this.last_name }, { - row: @row - column: @column - doc_id: @doc_id - }).should.equal true - done() + row: this.row, + column: this.column, + doc_id: this.doc_id + }).should.equal(true); + return done(); + }); - it "should increment the update-client-position metric at 0.1 frequency", -> - @metrics.inc.calledWith("editor.update-client-position", 0.1).should.equal true + return it("should increment the update-client-position metric at 0.1 frequency", function() { + return this.metrics.inc.calledWith("editor.update-client-position", 0.1).should.equal(true); + }); + }); - describe "with a logged in user who has no last_name set", -> - beforeEach -> - @client.ol_context = { - project_id: @project_id - first_name: @first_name = "Douglas" - last_name: undefined - email: @email = "joe@example.com" - user_id: @user_id = "user-id-123" - } - @WebsocketController.updateClientPosition @client, @update + describe("with a logged in user who has no last_name set", function() { + beforeEach(function() { + this.client.ol_context = { + project_id: this.project_id, + first_name: (this.first_name = "Douglas"), + last_name: undefined, + email: (this.email = "joe@example.com"), + user_id: (this.user_id = "user-id-123") + }; + this.WebsocketController.updateClientPosition(this.client, this.update); - @populatedCursorData = - doc_id: @doc_id, - id: @client.publicId - name: "#{@first_name}" - row: @row - column: @column - email: @email - user_id: @user_id + return this.populatedCursorData = { + doc_id: this.doc_id, + id: this.client.publicId, + name: `${this.first_name}`, + row: this.row, + column: this.column, + email: this.email, + user_id: this.user_id + }; + }); - it "should send the update to the project room with the user's name", -> - @WebsocketLoadBalancer.emitToRoom.calledWith(@project_id, "clientTracking.clientUpdated", @populatedCursorData).should.equal true + it("should send the update to the project room with the user's name", function() { + return this.WebsocketLoadBalancer.emitToRoom.calledWith(this.project_id, "clientTracking.clientUpdated", this.populatedCursorData).should.equal(true); + }); - it "should send the cursor data to the connected user manager", (done)-> - @ConnectedUsersManager.updateUserPosition.calledWith(@project_id, @client.publicId, { - _id: @user_id, - email: @email, - first_name: @first_name, + it("should send the cursor data to the connected user manager", function(done){ + this.ConnectedUsersManager.updateUserPosition.calledWith(this.project_id, this.client.publicId, { + _id: this.user_id, + email: this.email, + first_name: this.first_name, last_name: undefined }, { - row: @row - column: @column - doc_id: @doc_id - }).should.equal true - done() + row: this.row, + column: this.column, + doc_id: this.doc_id + }).should.equal(true); + return done(); + }); - it "should increment the update-client-position metric at 0.1 frequency", -> - @metrics.inc.calledWith("editor.update-client-position", 0.1).should.equal true + return it("should increment the update-client-position metric at 0.1 frequency", function() { + return this.metrics.inc.calledWith("editor.update-client-position", 0.1).should.equal(true); + }); + }); - describe "with a logged in user who has no first_name set", -> - beforeEach -> - @client.ol_context = { - project_id: @project_id - first_name: undefined - last_name: @last_name = "Adams" - email: @email = "joe@example.com" - user_id: @user_id = "user-id-123" - } - @WebsocketController.updateClientPosition @client, @update - - @populatedCursorData = - doc_id: @doc_id, - id: @client.publicId - name: "#{@last_name}" - row: @row - column: @column - email: @email - user_id: @user_id - - it "should send the update to the project room with the user's name", -> - @WebsocketLoadBalancer.emitToRoom.calledWith(@project_id, "clientTracking.clientUpdated", @populatedCursorData).should.equal true - - it "should send the cursor data to the connected user manager", (done)-> - @ConnectedUsersManager.updateUserPosition.calledWith(@project_id, @client.publicId, { - _id: @user_id, - email: @email, + describe("with a logged in user who has no first_name set", function() { + beforeEach(function() { + this.client.ol_context = { + project_id: this.project_id, first_name: undefined, - last_name: @last_name + last_name: (this.last_name = "Adams"), + email: (this.email = "joe@example.com"), + user_id: (this.user_id = "user-id-123") + }; + this.WebsocketController.updateClientPosition(this.client, this.update); + + return this.populatedCursorData = { + doc_id: this.doc_id, + id: this.client.publicId, + name: `${this.last_name}`, + row: this.row, + column: this.column, + email: this.email, + user_id: this.user_id + }; + }); + + it("should send the update to the project room with the user's name", function() { + return this.WebsocketLoadBalancer.emitToRoom.calledWith(this.project_id, "clientTracking.clientUpdated", this.populatedCursorData).should.equal(true); + }); + + it("should send the cursor data to the connected user manager", function(done){ + this.ConnectedUsersManager.updateUserPosition.calledWith(this.project_id, this.client.publicId, { + _id: this.user_id, + email: this.email, + first_name: undefined, + last_name: this.last_name }, { - row: @row - column: @column - doc_id: @doc_id - }).should.equal true - done() + row: this.row, + column: this.column, + doc_id: this.doc_id + }).should.equal(true); + return done(); + }); - it "should increment the update-client-position metric at 0.1 frequency", -> - @metrics.inc.calledWith("editor.update-client-position", 0.1).should.equal true - describe "with a logged in user who has no names set", -> - beforeEach -> - @client.ol_context = { - project_id: @project_id - first_name: undefined - last_name: undefined - email: @email = "joe@example.com" - user_id: @user_id = "user-id-123" - } - @WebsocketController.updateClientPosition @client, @update + return it("should increment the update-client-position metric at 0.1 frequency", function() { + return this.metrics.inc.calledWith("editor.update-client-position", 0.1).should.equal(true); + }); + }); + describe("with a logged in user who has no names set", function() { + beforeEach(function() { + this.client.ol_context = { + project_id: this.project_id, + first_name: undefined, + last_name: undefined, + email: (this.email = "joe@example.com"), + user_id: (this.user_id = "user-id-123") + }; + return this.WebsocketController.updateClientPosition(this.client, this.update); + }); - it "should send the update to the project name with no name", -> - @WebsocketLoadBalancer.emitToRoom - .calledWith(@project_id, "clientTracking.clientUpdated", { - doc_id: @doc_id, - id: @client.publicId, - user_id: @user_id, + return it("should send the update to the project name with no name", function() { + return this.WebsocketLoadBalancer.emitToRoom + .calledWith(this.project_id, "clientTracking.clientUpdated", { + doc_id: this.doc_id, + id: this.client.publicId, + user_id: this.user_id, name: "", - row: @row, - column: @column, - email: @email + row: this.row, + column: this.column, + email: this.email }) - .should.equal true + .should.equal(true); + }); + }); - describe "with an anonymous user", -> - beforeEach -> - @client.ol_context = { - project_id: @project_id - } - @WebsocketController.updateClientPosition @client, @update + describe("with an anonymous user", function() { + beforeEach(function() { + this.client.ol_context = { + project_id: this.project_id + }; + return this.WebsocketController.updateClientPosition(this.client, this.update); + }); - it "should send the update to the project room with no name", -> - @WebsocketLoadBalancer.emitToRoom - .calledWith(@project_id, "clientTracking.clientUpdated", { - doc_id: @doc_id, - id: @client.publicId - name: "" - row: @row - column: @column + it("should send the update to the project room with no name", function() { + return this.WebsocketLoadBalancer.emitToRoom + .calledWith(this.project_id, "clientTracking.clientUpdated", { + doc_id: this.doc_id, + id: this.client.publicId, + name: "", + row: this.row, + column: this.column }) - .should.equal true + .should.equal(true); + }); - it "should not send cursor data to the connected user manager", (done)-> - @ConnectedUsersManager.updateUserPosition.called.should.equal false - done() + return it("should not send cursor data to the connected user manager", function(done){ + this.ConnectedUsersManager.updateUserPosition.called.should.equal(false); + return done(); + }); + }); - describe "when the client has disconnected", -> - beforeEach -> - @client.disconnected = true - @AuthorizationManager.assertClientCanViewProjectAndDoc = sinon.stub() - @WebsocketController.updateClientPosition @client, @update, @callback + return describe("when the client has disconnected", function() { + beforeEach(function() { + this.client.disconnected = true; + this.AuthorizationManager.assertClientCanViewProjectAndDoc = sinon.stub(); + return this.WebsocketController.updateClientPosition(this.client, this.update, this.callback); + }); - it "should call the callback with no details", -> - expect(@callback.args[0]).to.deep.equal([]) + it("should call the callback with no details", function() { + return expect(this.callback.args[0]).to.deep.equal([]); + }); - it "should not check permissions", -> - expect(@AuthorizationManager.assertClientCanViewProjectAndDoc.called).to.equal(false) + return it("should not check permissions", function() { + return expect(this.AuthorizationManager.assertClientCanViewProjectAndDoc.called).to.equal(false); + }); + }); + }); - describe "applyOtUpdate", -> - beforeEach -> - @update = {op: {p: 12, t: "foo"}} - @client.ol_context.user_id = @user_id - @client.ol_context.project_id = @project_id - @WebsocketController._assertClientCanApplyUpdate = sinon.stub().yields() - @DocumentUpdaterManager.queueChange = sinon.stub().callsArg(3) + describe("applyOtUpdate", function() { + beforeEach(function() { + this.update = {op: {p: 12, t: "foo"}}; + this.client.ol_context.user_id = this.user_id; + this.client.ol_context.project_id = this.project_id; + this.WebsocketController._assertClientCanApplyUpdate = sinon.stub().yields(); + return this.DocumentUpdaterManager.queueChange = sinon.stub().callsArg(3); + }); - describe "succesfully", -> - beforeEach -> - @WebsocketController.applyOtUpdate @client, @doc_id, @update, @callback + describe("succesfully", function() { + beforeEach(function() { + return this.WebsocketController.applyOtUpdate(this.client, this.doc_id, this.update, this.callback); + }); - it "should set the source of the update to the client id", -> - @update.meta.source.should.equal @client.publicId + it("should set the source of the update to the client id", function() { + return this.update.meta.source.should.equal(this.client.publicId); + }); - it "should set the user_id of the update to the user id", -> - @update.meta.user_id.should.equal @user_id + it("should set the user_id of the update to the user id", function() { + return this.update.meta.user_id.should.equal(this.user_id); + }); - it "should queue the update", -> - @DocumentUpdaterManager.queueChange - .calledWith(@project_id, @doc_id, @update) - .should.equal true + it("should queue the update", function() { + return this.DocumentUpdaterManager.queueChange + .calledWith(this.project_id, this.doc_id, this.update) + .should.equal(true); + }); - it "should call the callback", -> - @callback.called.should.equal true + it("should call the callback", function() { + return this.callback.called.should.equal(true); + }); - it "should increment the doc updates", -> - @metrics.inc.calledWith("editor.doc-update").should.equal true + return it("should increment the doc updates", function() { + return this.metrics.inc.calledWith("editor.doc-update").should.equal(true); + }); + }); - describe "unsuccessfully", -> - beforeEach -> - @client.disconnect = sinon.stub() - @DocumentUpdaterManager.queueChange = sinon.stub().callsArgWith(3, @error = new Error("Something went wrong")) - @WebsocketController.applyOtUpdate @client, @doc_id, @update, @callback + describe("unsuccessfully", function() { + beforeEach(function() { + this.client.disconnect = sinon.stub(); + this.DocumentUpdaterManager.queueChange = sinon.stub().callsArgWith(3, (this.error = new Error("Something went wrong"))); + return this.WebsocketController.applyOtUpdate(this.client, this.doc_id, this.update, this.callback); + }); - it "should disconnect the client", -> - @client.disconnect.called.should.equal true + it("should disconnect the client", function() { + return this.client.disconnect.called.should.equal(true); + }); - it "should log an error", -> - @logger.error.called.should.equal true + it("should log an error", function() { + return this.logger.error.called.should.equal(true); + }); - it "should call the callback with the error", -> - @callback.calledWith(@error).should.equal true + return it("should call the callback with the error", function() { + return this.callback.calledWith(this.error).should.equal(true); + }); + }); - describe "when not authorized", -> - beforeEach -> - @client.disconnect = sinon.stub() - @WebsocketController._assertClientCanApplyUpdate = sinon.stub().yields(@error = new Error("not authorized")) - @WebsocketController.applyOtUpdate @client, @doc_id, @update, @callback + describe("when not authorized", function() { + beforeEach(function() { + this.client.disconnect = sinon.stub(); + this.WebsocketController._assertClientCanApplyUpdate = sinon.stub().yields(this.error = new Error("not authorized")); + return this.WebsocketController.applyOtUpdate(this.client, this.doc_id, this.update, this.callback); + }); - # This happens in a setTimeout to allow the client a chance to receive the error first. - # I'm not sure how to unit test, but it is acceptance tested. - # it "should disconnect the client", -> - # @client.disconnect.called.should.equal true + // This happens in a setTimeout to allow the client a chance to receive the error first. + // I'm not sure how to unit test, but it is acceptance tested. + // it "should disconnect the client", -> + // @client.disconnect.called.should.equal true - it "should log a warning", -> - @logger.warn.called.should.equal true + it("should log a warning", function() { + return this.logger.warn.called.should.equal(true); + }); - it "should call the callback with the error", -> - @callback.calledWith(@error).should.equal true + return it("should call the callback with the error", function() { + return this.callback.calledWith(this.error).should.equal(true); + }); + }); - describe "update_too_large", -> - beforeEach (done) -> - @client.disconnect = sinon.stub() - @client.emit = sinon.stub() - @client.ol_context.user_id = @user_id - @client.ol_context.project_id = @project_id - error = new Error("update is too large") - error.updateSize = 7372835 - @DocumentUpdaterManager.queueChange = sinon.stub().callsArgWith(3, error) - @WebsocketController.applyOtUpdate @client, @doc_id, @update, @callback - setTimeout -> - done() - , 1 + return describe("update_too_large", function() { + beforeEach(function(done) { + this.client.disconnect = sinon.stub(); + this.client.emit = sinon.stub(); + this.client.ol_context.user_id = this.user_id; + this.client.ol_context.project_id = this.project_id; + const error = new Error("update is too large"); + error.updateSize = 7372835; + this.DocumentUpdaterManager.queueChange = sinon.stub().callsArgWith(3, error); + this.WebsocketController.applyOtUpdate(this.client, this.doc_id, this.update, this.callback); + return setTimeout(() => done() + , 1); + }); - it "should call the callback with no error", -> - @callback.called.should.equal true - @callback.args[0].should.deep.equal [] + it("should call the callback with no error", function() { + this.callback.called.should.equal(true); + return this.callback.args[0].should.deep.equal([]); + }); - it "should log a warning with the size and context", -> - @logger.warn.called.should.equal true - @logger.warn.args[0].should.deep.equal [{ - @user_id, @project_id, @doc_id, updateSize: 7372835 - }, 'update is too large'] + it("should log a warning with the size and context", function() { + this.logger.warn.called.should.equal(true); + return this.logger.warn.args[0].should.deep.equal([{ + user_id: this.user_id, project_id: this.project_id, doc_id: this.doc_id, updateSize: 7372835 + }, 'update is too large']); + }); - describe "after 100ms", -> - beforeEach (done) -> - setTimeout done, 100 + describe("after 100ms", function() { + beforeEach(done => setTimeout(done, 100)); - it "should send an otUpdateError the client", -> - @client.emit.calledWith('otUpdateError').should.equal true + it("should send an otUpdateError the client", function() { + return this.client.emit.calledWith('otUpdateError').should.equal(true); + }); - it "should disconnect the client", -> - @client.disconnect.called.should.equal true + return it("should disconnect the client", function() { + return this.client.disconnect.called.should.equal(true); + }); + }); - describe "when the client disconnects during the next 100ms", -> - beforeEach (done) -> - @client.disconnected = true - setTimeout done, 100 + return describe("when the client disconnects during the next 100ms", function() { + beforeEach(function(done) { + this.client.disconnected = true; + return setTimeout(done, 100); + }); - it "should not send an otUpdateError the client", -> - @client.emit.calledWith('otUpdateError').should.equal false + it("should not send an otUpdateError the client", function() { + return this.client.emit.calledWith('otUpdateError').should.equal(false); + }); - it "should not disconnect the client", -> - @client.disconnect.called.should.equal false + it("should not disconnect the client", function() { + return this.client.disconnect.called.should.equal(false); + }); - it "should increment the editor.doc-update.disconnected metric with a status", -> - expect(@metrics.inc.calledWith('editor.doc-update.disconnected', 1, {status:'at-otUpdateError'})).to.equal(true) + return it("should increment the editor.doc-update.disconnected metric with a status", function() { + return expect(this.metrics.inc.calledWith('editor.doc-update.disconnected', 1, {status:'at-otUpdateError'})).to.equal(true); + }); + }); + }); + }); - describe "_assertClientCanApplyUpdate", -> - beforeEach -> - @edit_update = { op: [{i: "foo", p: 42}, {c: "bar", p: 132}] } # comments may still be in an edit op - @comment_update = { op: [{c: "bar", p: 132}] } - @AuthorizationManager.assertClientCanEditProjectAndDoc = sinon.stub() - @AuthorizationManager.assertClientCanViewProjectAndDoc = sinon.stub() + return describe("_assertClientCanApplyUpdate", function() { + beforeEach(function() { + this.edit_update = { op: [{i: "foo", p: 42}, {c: "bar", p: 132}] }; // comments may still be in an edit op + this.comment_update = { op: [{c: "bar", p: 132}] }; + this.AuthorizationManager.assertClientCanEditProjectAndDoc = sinon.stub(); + return this.AuthorizationManager.assertClientCanViewProjectAndDoc = sinon.stub(); + }); - describe "with a read-write client", -> - it "should return successfully", (done) -> - @AuthorizationManager.assertClientCanEditProjectAndDoc.yields(null) - @WebsocketController._assertClientCanApplyUpdate @client, @doc_id, @edit_update, (error) -> - expect(error).to.be.null - done() + describe("with a read-write client", () => it("should return successfully", function(done) { + this.AuthorizationManager.assertClientCanEditProjectAndDoc.yields(null); + return this.WebsocketController._assertClientCanApplyUpdate(this.client, this.doc_id, this.edit_update, function(error) { + expect(error).to.be.null; + return done(); + }); + })); - describe "with a read-only client and an edit op", -> - it "should return an error", (done) -> - @AuthorizationManager.assertClientCanEditProjectAndDoc.yields(new Error("not authorized")) - @AuthorizationManager.assertClientCanViewProjectAndDoc.yields(null) - @WebsocketController._assertClientCanApplyUpdate @client, @doc_id, @edit_update, (error) -> - expect(error.message).to.equal "not authorized" - done() + describe("with a read-only client and an edit op", () => it("should return an error", function(done) { + this.AuthorizationManager.assertClientCanEditProjectAndDoc.yields(new Error("not authorized")); + this.AuthorizationManager.assertClientCanViewProjectAndDoc.yields(null); + return this.WebsocketController._assertClientCanApplyUpdate(this.client, this.doc_id, this.edit_update, function(error) { + expect(error.message).to.equal("not authorized"); + return done(); + }); + })); - describe "with a read-only client and a comment op", -> - it "should return successfully", (done) -> - @AuthorizationManager.assertClientCanEditProjectAndDoc.yields(new Error("not authorized")) - @AuthorizationManager.assertClientCanViewProjectAndDoc.yields(null) - @WebsocketController._assertClientCanApplyUpdate @client, @doc_id, @comment_update, (error) -> - expect(error).to.be.null - done() + describe("with a read-only client and a comment op", () => it("should return successfully", function(done) { + this.AuthorizationManager.assertClientCanEditProjectAndDoc.yields(new Error("not authorized")); + this.AuthorizationManager.assertClientCanViewProjectAndDoc.yields(null); + return this.WebsocketController._assertClientCanApplyUpdate(this.client, this.doc_id, this.comment_update, function(error) { + expect(error).to.be.null; + return done(); + }); + })); - describe "with a totally unauthorized client", -> - it "should return an error", (done) -> - @AuthorizationManager.assertClientCanEditProjectAndDoc.yields(new Error("not authorized")) - @AuthorizationManager.assertClientCanViewProjectAndDoc.yields(new Error("not authorized")) - @WebsocketController._assertClientCanApplyUpdate @client, @doc_id, @comment_update, (error) -> - expect(error.message).to.equal "not authorized" - done() + return describe("with a totally unauthorized client", () => it("should return an error", function(done) { + this.AuthorizationManager.assertClientCanEditProjectAndDoc.yields(new Error("not authorized")); + this.AuthorizationManager.assertClientCanViewProjectAndDoc.yields(new Error("not authorized")); + return this.WebsocketController._assertClientCanApplyUpdate(this.client, this.doc_id, this.comment_update, function(error) { + expect(error.message).to.equal("not authorized"); + return done(); + }); + })); + }); +}); diff --git a/services/real-time/test/unit/coffee/WebsocketLoadBalancerTests.js b/services/real-time/test/unit/coffee/WebsocketLoadBalancerTests.js index b2441cd6d0..5b9ab079fb 100644 --- a/services/real-time/test/unit/coffee/WebsocketLoadBalancerTests.js +++ b/services/real-time/test/unit/coffee/WebsocketLoadBalancerTests.js @@ -1,161 +1,204 @@ -SandboxedModule = require('sandboxed-module') -sinon = require('sinon') -require('chai').should() -modulePath = require('path').join __dirname, '../../../app/js/WebsocketLoadBalancer' +/* + * decaffeinate suggestions: + * DS101: Remove unnecessary use of Array.from + * DS102: Remove unnecessary code created because of implicit returns + * Full docs: https://github.com/decaffeinate/decaffeinate/blob/master/docs/suggestions.md + */ +const SandboxedModule = require('sandboxed-module'); +const sinon = require('sinon'); +require('chai').should(); +const modulePath = require('path').join(__dirname, '../../../app/js/WebsocketLoadBalancer'); -describe "WebsocketLoadBalancer", -> - beforeEach -> - @rclient = {} - @RoomEvents = {on: sinon.stub()} - @WebsocketLoadBalancer = SandboxedModule.require modulePath, requires: - "./RedisClientManager": +describe("WebsocketLoadBalancer", function() { + beforeEach(function() { + this.rclient = {}; + this.RoomEvents = {on: sinon.stub()}; + this.WebsocketLoadBalancer = SandboxedModule.require(modulePath, { requires: { + "./RedisClientManager": { createClientList: () => [] - "logger-sharelatex": @logger = { log: sinon.stub(), error: sinon.stub() } - "./SafeJsonParse": @SafeJsonParse = - parse: (data, cb) => cb null, JSON.parse(data) - "./EventLogger": {checkEventOrder: sinon.stub()} - "./HealthCheckManager": {check: sinon.stub()} - "./RoomManager" : @RoomManager = {eventSource: sinon.stub().returns @RoomEvents} - "./ChannelManager": @ChannelManager = {publish: sinon.stub()} - "./ConnectedUsersManager": @ConnectedUsersManager = {refreshClient: sinon.stub()} - @io = {} - @WebsocketLoadBalancer.rclientPubList = [{publish: sinon.stub()}] - @WebsocketLoadBalancer.rclientSubList = [{ - subscribe: sinon.stub() + }, + "logger-sharelatex": (this.logger = { log: sinon.stub(), error: sinon.stub() }), + "./SafeJsonParse": (this.SafeJsonParse = + {parse: (data, cb) => cb(null, JSON.parse(data))}), + "./EventLogger": {checkEventOrder: sinon.stub()}, + "./HealthCheckManager": {check: sinon.stub()}, + "./RoomManager" : (this.RoomManager = {eventSource: sinon.stub().returns(this.RoomEvents)}), + "./ChannelManager": (this.ChannelManager = {publish: sinon.stub()}), + "./ConnectedUsersManager": (this.ConnectedUsersManager = {refreshClient: sinon.stub()}) + } + }); + this.io = {}; + this.WebsocketLoadBalancer.rclientPubList = [{publish: sinon.stub()}]; + this.WebsocketLoadBalancer.rclientSubList = [{ + subscribe: sinon.stub(), on: sinon.stub() - }] + }]; - @room_id = "room-id" - @message = "otUpdateApplied" - @payload = ["argument one", 42] + this.room_id = "room-id"; + this.message = "otUpdateApplied"; + return this.payload = ["argument one", 42];}); - describe "emitToRoom", -> - beforeEach -> - @WebsocketLoadBalancer.emitToRoom(@room_id, @message, @payload...) + describe("emitToRoom", function() { + beforeEach(function() { + return this.WebsocketLoadBalancer.emitToRoom(this.room_id, this.message, ...Array.from(this.payload)); + }); - it "should publish the message to redis", -> - @ChannelManager.publish - .calledWith(@WebsocketLoadBalancer.rclientPubList[0], "editor-events", @room_id, JSON.stringify( - room_id: @room_id, - message: @message - payload: @payload - )) - .should.equal true + return it("should publish the message to redis", function() { + return this.ChannelManager.publish + .calledWith(this.WebsocketLoadBalancer.rclientPubList[0], "editor-events", this.room_id, JSON.stringify({ + room_id: this.room_id, + message: this.message, + payload: this.payload + })) + .should.equal(true); + }); + }); - describe "emitToAll", -> - beforeEach -> - @WebsocketLoadBalancer.emitToRoom = sinon.stub() - @WebsocketLoadBalancer.emitToAll @message, @payload... + describe("emitToAll", function() { + beforeEach(function() { + this.WebsocketLoadBalancer.emitToRoom = sinon.stub(); + return this.WebsocketLoadBalancer.emitToAll(this.message, ...Array.from(this.payload)); + }); - it "should emit to the room 'all'", -> - @WebsocketLoadBalancer.emitToRoom - .calledWith("all", @message, @payload...) - .should.equal true + return it("should emit to the room 'all'", function() { + return this.WebsocketLoadBalancer.emitToRoom + .calledWith("all", this.message, ...Array.from(this.payload)) + .should.equal(true); + }); + }); - describe "listenForEditorEvents", -> - beforeEach -> - @WebsocketLoadBalancer._processEditorEvent = sinon.stub() - @WebsocketLoadBalancer.listenForEditorEvents() + describe("listenForEditorEvents", function() { + beforeEach(function() { + this.WebsocketLoadBalancer._processEditorEvent = sinon.stub(); + return this.WebsocketLoadBalancer.listenForEditorEvents(); + }); - it "should subscribe to the editor-events channel", -> - @WebsocketLoadBalancer.rclientSubList[0].subscribe + it("should subscribe to the editor-events channel", function() { + return this.WebsocketLoadBalancer.rclientSubList[0].subscribe .calledWith("editor-events") - .should.equal true + .should.equal(true); + }); - it "should process the events with _processEditorEvent", -> - @WebsocketLoadBalancer.rclientSubList[0].on + return it("should process the events with _processEditorEvent", function() { + return this.WebsocketLoadBalancer.rclientSubList[0].on .calledWith("message", sinon.match.func) - .should.equal true + .should.equal(true); + }); + }); - describe "_processEditorEvent", -> - describe "with bad JSON", -> - beforeEach -> - @isRestrictedUser = false - @SafeJsonParse.parse = sinon.stub().callsArgWith 1, new Error("oops") - @WebsocketLoadBalancer._processEditorEvent(@io, "editor-events", "blah") + return describe("_processEditorEvent", function() { + describe("with bad JSON", function() { + beforeEach(function() { + this.isRestrictedUser = false; + this.SafeJsonParse.parse = sinon.stub().callsArgWith(1, new Error("oops")); + return this.WebsocketLoadBalancer._processEditorEvent(this.io, "editor-events", "blah"); + }); - it "should log an error", -> - @logger.error.called.should.equal true + return it("should log an error", function() { + return this.logger.error.called.should.equal(true); + }); + }); - describe "with a designated room", -> - beforeEach -> - @io.sockets = + describe("with a designated room", function() { + beforeEach(function() { + this.io.sockets = { clients: sinon.stub().returns([ - {id: 'client-id-1', emit: @emit1 = sinon.stub(), ol_context: {}} - {id: 'client-id-2', emit: @emit2 = sinon.stub(), ol_context: {}} - {id: 'client-id-1', emit: @emit3 = sinon.stub(), ol_context: {}} # duplicate client + {id: 'client-id-1', emit: (this.emit1 = sinon.stub()), ol_context: {}}, + {id: 'client-id-2', emit: (this.emit2 = sinon.stub()), ol_context: {}}, + {id: 'client-id-1', emit: (this.emit3 = sinon.stub()), ol_context: {}} // duplicate client ]) - data = JSON.stringify - room_id: @room_id - message: @message - payload: @payload - @WebsocketLoadBalancer._processEditorEvent(@io, "editor-events", data) + }; + const data = JSON.stringify({ + room_id: this.room_id, + message: this.message, + payload: this.payload + }); + return this.WebsocketLoadBalancer._processEditorEvent(this.io, "editor-events", data); + }); - it "should send the message to all (unique) clients in the room", -> - @io.sockets.clients - .calledWith(@room_id) - .should.equal true - @emit1.calledWith(@message, @payload...).should.equal true - @emit2.calledWith(@message, @payload...).should.equal true - @emit3.called.should.equal false # duplicate client should be ignored + return it("should send the message to all (unique) clients in the room", function() { + this.io.sockets.clients + .calledWith(this.room_id) + .should.equal(true); + this.emit1.calledWith(this.message, ...Array.from(this.payload)).should.equal(true); + this.emit2.calledWith(this.message, ...Array.from(this.payload)).should.equal(true); + return this.emit3.called.should.equal(false); + }); + }); // duplicate client should be ignored - describe "with a designated room, and restricted clients, not restricted message", -> - beforeEach -> - @io.sockets = + describe("with a designated room, and restricted clients, not restricted message", function() { + beforeEach(function() { + this.io.sockets = { clients: sinon.stub().returns([ - {id: 'client-id-1', emit: @emit1 = sinon.stub(), ol_context: {}} - {id: 'client-id-2', emit: @emit2 = sinon.stub(), ol_context: {}} - {id: 'client-id-1', emit: @emit3 = sinon.stub(), ol_context: {}} # duplicate client - {id: 'client-id-4', emit: @emit4 = sinon.stub(), ol_context: {is_restricted_user: true}} + {id: 'client-id-1', emit: (this.emit1 = sinon.stub()), ol_context: {}}, + {id: 'client-id-2', emit: (this.emit2 = sinon.stub()), ol_context: {}}, + {id: 'client-id-1', emit: (this.emit3 = sinon.stub()), ol_context: {}}, // duplicate client + {id: 'client-id-4', emit: (this.emit4 = sinon.stub()), ol_context: {is_restricted_user: true}} ]) - data = JSON.stringify - room_id: @room_id - message: @message - payload: @payload - @WebsocketLoadBalancer._processEditorEvent(@io, "editor-events", data) + }; + const data = JSON.stringify({ + room_id: this.room_id, + message: this.message, + payload: this.payload + }); + return this.WebsocketLoadBalancer._processEditorEvent(this.io, "editor-events", data); + }); - it "should send the message to all (unique) clients in the room", -> - @io.sockets.clients - .calledWith(@room_id) - .should.equal true - @emit1.calledWith(@message, @payload...).should.equal true - @emit2.calledWith(@message, @payload...).should.equal true - @emit3.called.should.equal false # duplicate client should be ignored - @emit4.called.should.equal true # restricted client, but should be called + return it("should send the message to all (unique) clients in the room", function() { + this.io.sockets.clients + .calledWith(this.room_id) + .should.equal(true); + this.emit1.calledWith(this.message, ...Array.from(this.payload)).should.equal(true); + this.emit2.calledWith(this.message, ...Array.from(this.payload)).should.equal(true); + this.emit3.called.should.equal(false); // duplicate client should be ignored + return this.emit4.called.should.equal(true); + }); + }); // restricted client, but should be called - describe "with a designated room, and restricted clients, restricted message", -> - beforeEach -> - @io.sockets = + describe("with a designated room, and restricted clients, restricted message", function() { + beforeEach(function() { + this.io.sockets = { clients: sinon.stub().returns([ - {id: 'client-id-1', emit: @emit1 = sinon.stub(), ol_context: {}} - {id: 'client-id-2', emit: @emit2 = sinon.stub(), ol_context: {}} - {id: 'client-id-1', emit: @emit3 = sinon.stub(), ol_context: {}} # duplicate client - {id: 'client-id-4', emit: @emit4 = sinon.stub(), ol_context: {is_restricted_user: true}} + {id: 'client-id-1', emit: (this.emit1 = sinon.stub()), ol_context: {}}, + {id: 'client-id-2', emit: (this.emit2 = sinon.stub()), ol_context: {}}, + {id: 'client-id-1', emit: (this.emit3 = sinon.stub()), ol_context: {}}, // duplicate client + {id: 'client-id-4', emit: (this.emit4 = sinon.stub()), ol_context: {is_restricted_user: true}} ]) - data = JSON.stringify - room_id: @room_id - message: @restrictedMessage = 'new-comment' - payload: @payload - @WebsocketLoadBalancer._processEditorEvent(@io, "editor-events", data) + }; + const data = JSON.stringify({ + room_id: this.room_id, + message: (this.restrictedMessage = 'new-comment'), + payload: this.payload + }); + return this.WebsocketLoadBalancer._processEditorEvent(this.io, "editor-events", data); + }); - it "should send the message to all (unique) clients in the room, who are not restricted", -> - @io.sockets.clients - .calledWith(@room_id) - .should.equal true - @emit1.calledWith(@restrictedMessage, @payload...).should.equal true - @emit2.calledWith(@restrictedMessage, @payload...).should.equal true - @emit3.called.should.equal false # duplicate client should be ignored - @emit4.called.should.equal false # restricted client, should not be called + return it("should send the message to all (unique) clients in the room, who are not restricted", function() { + this.io.sockets.clients + .calledWith(this.room_id) + .should.equal(true); + this.emit1.calledWith(this.restrictedMessage, ...Array.from(this.payload)).should.equal(true); + this.emit2.calledWith(this.restrictedMessage, ...Array.from(this.payload)).should.equal(true); + this.emit3.called.should.equal(false); // duplicate client should be ignored + return this.emit4.called.should.equal(false); + }); + }); // restricted client, should not be called - describe "when emitting to all", -> - beforeEach -> - @io.sockets = - emit: @emit = sinon.stub() - data = JSON.stringify - room_id: "all" - message: @message - payload: @payload - @WebsocketLoadBalancer._processEditorEvent(@io, "editor-events", data) + return describe("when emitting to all", function() { + beforeEach(function() { + this.io.sockets = + {emit: (this.emit = sinon.stub())}; + const data = JSON.stringify({ + room_id: "all", + message: this.message, + payload: this.payload + }); + return this.WebsocketLoadBalancer._processEditorEvent(this.io, "editor-events", data); + }); - it "should send the message to all clients", -> - @emit.calledWith(@message, @payload...).should.equal true + return it("should send the message to all clients", function() { + return this.emit.calledWith(this.message, ...Array.from(this.payload)).should.equal(true); + }); + }); + }); +}); diff --git a/services/real-time/test/unit/coffee/helpers/MockClient.js b/services/real-time/test/unit/coffee/helpers/MockClient.js index 497928132a..bdf711ac8d 100644 --- a/services/real-time/test/unit/coffee/helpers/MockClient.js +++ b/services/real-time/test/unit/coffee/helpers/MockClient.js @@ -1,13 +1,16 @@ -sinon = require('sinon') +let MockClient; +const sinon = require('sinon'); -idCounter = 0 +let idCounter = 0; -module.exports = class MockClient - constructor: () -> - @ol_context = {} - @join = sinon.stub() - @emit = sinon.stub() - @disconnect = sinon.stub() - @id = idCounter++ - @publicId = idCounter++ - disconnect: () -> +module.exports = (MockClient = class MockClient { + constructor() { + this.ol_context = {}; + this.join = sinon.stub(); + this.emit = sinon.stub(); + this.disconnect = sinon.stub(); + this.id = idCounter++; + this.publicId = idCounter++; + } + disconnect() {} +}); From 93697a8c5c2d7b6608606a31f52b29a1070e817b Mon Sep 17 00:00:00 2001 From: decaffeinate Date: Tue, 23 Jun 2020 18:30:03 +0100 Subject: [PATCH 375/491] decaffeinate: Run post-processing cleanups on AuthorizationManagerTests.coffee and 13 other files --- .../unit/coffee/AuthorizationManagerTests.js | 30 ++++++---- .../test/unit/coffee/ChannelManagerTests.js | 10 +++- .../unit/coffee/ConnectedUsersManagerTests.js | 10 +++- .../coffee/DocumentUpdaterControllerTests.js | 6 ++ .../coffee/DocumentUpdaterManagerTests.js | 7 +++ .../test/unit/coffee/DrainManagerTests.js | 6 ++ .../test/unit/coffee/EventLoggerTests.js | 11 +++- .../test/unit/coffee/RoomManagerTests.js | 17 ++++-- .../test/unit/coffee/SafeJsonParseTest.js | 12 +++- .../test/unit/coffee/SessionSocketsTests.js | 22 ++++--- .../test/unit/coffee/WebApiManagerTests.js | 6 ++ .../unit/coffee/WebsocketControllerTests.js | 58 +++++++++++-------- .../unit/coffee/WebsocketLoadBalancerTests.js | 5 ++ .../test/unit/coffee/helpers/MockClient.js | 6 ++ 14 files changed, 148 insertions(+), 58 deletions(-) diff --git a/services/real-time/test/unit/coffee/AuthorizationManagerTests.js b/services/real-time/test/unit/coffee/AuthorizationManagerTests.js index 626428ed61..d3aa6be9fa 100644 --- a/services/real-time/test/unit/coffee/AuthorizationManagerTests.js +++ b/services/real-time/test/unit/coffee/AuthorizationManagerTests.js @@ -1,3 +1,9 @@ +/* eslint-disable + no-return-assign, + no-unused-vars, +*/ +// TODO: This file was created by bulk-decaffeinate. +// Fix any style issues and re-enable lint. /* * decaffeinate suggestions: * DS102: Remove unnecessary code created because of implicit returns @@ -23,7 +29,7 @@ describe('AuthorizationManager', function() { describe("assertClientCanViewProject", function() { it("should allow the readOnly privilegeLevel", function(done) { this.client.ol_context.privilege_level = "readOnly"; - return this.AuthorizationManager.assertClientCanViewProject(this.client, function(error) { + return this.AuthorizationManager.assertClientCanViewProject(this.client, (error) => { expect(error).to.be.null; return done(); }); @@ -31,7 +37,7 @@ describe('AuthorizationManager', function() { it("should allow the readAndWrite privilegeLevel", function(done) { this.client.ol_context.privilege_level = "readAndWrite"; - return this.AuthorizationManager.assertClientCanViewProject(this.client, function(error) { + return this.AuthorizationManager.assertClientCanViewProject(this.client, (error) => { expect(error).to.be.null; return done(); }); @@ -39,7 +45,7 @@ describe('AuthorizationManager', function() { it("should allow the owner privilegeLevel", function(done) { this.client.ol_context.privilege_level = "owner"; - return this.AuthorizationManager.assertClientCanViewProject(this.client, function(error) { + return this.AuthorizationManager.assertClientCanViewProject(this.client, (error) => { expect(error).to.be.null; return done(); }); @@ -47,7 +53,7 @@ describe('AuthorizationManager', function() { return it("should return an error with any other privilegeLevel", function(done) { this.client.ol_context.privilege_level = "unknown"; - return this.AuthorizationManager.assertClientCanViewProject(this.client, function(error) { + return this.AuthorizationManager.assertClientCanViewProject(this.client, (error) => { error.message.should.equal("not authorized"); return done(); }); @@ -57,7 +63,7 @@ describe('AuthorizationManager', function() { describe("assertClientCanEditProject", function() { it("should not allow the readOnly privilegeLevel", function(done) { this.client.ol_context.privilege_level = "readOnly"; - return this.AuthorizationManager.assertClientCanEditProject(this.client, function(error) { + return this.AuthorizationManager.assertClientCanEditProject(this.client, (error) => { error.message.should.equal("not authorized"); return done(); }); @@ -65,7 +71,7 @@ describe('AuthorizationManager', function() { it("should allow the readAndWrite privilegeLevel", function(done) { this.client.ol_context.privilege_level = "readAndWrite"; - return this.AuthorizationManager.assertClientCanEditProject(this.client, function(error) { + return this.AuthorizationManager.assertClientCanEditProject(this.client, (error) => { expect(error).to.be.null; return done(); }); @@ -73,7 +79,7 @@ describe('AuthorizationManager', function() { it("should allow the owner privilegeLevel", function(done) { this.client.ol_context.privilege_level = "owner"; - return this.AuthorizationManager.assertClientCanEditProject(this.client, function(error) { + return this.AuthorizationManager.assertClientCanEditProject(this.client, (error) => { expect(error).to.be.null; return done(); }); @@ -81,7 +87,7 @@ describe('AuthorizationManager', function() { return it("should return an error with any other privilegeLevel", function(done) { this.client.ol_context.privilege_level = "unknown"; - return this.AuthorizationManager.assertClientCanEditProject(this.client, function(error) { + return this.AuthorizationManager.assertClientCanEditProject(this.client, (error) => { error.message.should.equal("not authorized"); return done(); }); @@ -121,9 +127,9 @@ describe('AuthorizationManager', function() { return this.client.ol_context.privilege_level = "readOnly"; }); - describe("and not authorised at the document level", () => it("should not allow access", function() { + describe("and not authorised at the document level", function() { return it("should not allow access", function() { return this.AuthorizationManager.assertClientCanViewProjectAndDoc(this.client, this.doc_id, err => err.message.should.equal("not authorized")); - })); + }); }); describe("and authorised at the document level", function() { beforeEach(function(done) { @@ -183,9 +189,9 @@ describe('AuthorizationManager', function() { return this.client.ol_context.privilege_level = "readAndWrite"; }); - describe("and not authorised at the document level", () => it("should not allow access", function() { + describe("and not authorised at the document level", function() { return it("should not allow access", function() { return this.AuthorizationManager.assertClientCanEditProjectAndDoc(this.client, this.doc_id, err => err.message.should.equal("not authorized")); - })); + }); }); describe("and authorised at the document level", function() { beforeEach(function(done) { diff --git a/services/real-time/test/unit/coffee/ChannelManagerTests.js b/services/real-time/test/unit/coffee/ChannelManagerTests.js index 5c148451d0..1b71565975 100644 --- a/services/real-time/test/unit/coffee/ChannelManagerTests.js +++ b/services/real-time/test/unit/coffee/ChannelManagerTests.js @@ -1,3 +1,9 @@ +/* eslint-disable + no-return-assign, + no-unused-vars, +*/ +// TODO: This file was created by bulk-decaffeinate. +// Fix any style issues and re-enable lint. /* * decaffeinate suggestions: * DS102: Remove unnecessary code created because of implicit returns @@ -86,7 +92,7 @@ describe('ChannelManager', function() { .onSecondCall().resolves(); this.first = this.ChannelManager.subscribe(this.rclient, "applied-ops", "1234567890abcdef"); // ignore error - this.first.catch((function(){})); + this.first.catch((() => {})); expect(this.ChannelManager.getClientMapEntry(this.rclient).get("applied-ops:1234567890abcdef")).to.equal(this.first); this.rclient.unsubscribe = sinon.stub().resolves(); @@ -173,7 +179,7 @@ describe('ChannelManager', function() { beforeEach(function(done) { this.rclient.subscribe = sinon.stub().resolves(); this.ChannelManager.subscribe(this.rclient, "applied-ops", "1234567890abcdef"); - let rejectSubscribe = undefined; + let rejectSubscribe; this.rclient.unsubscribe = () => new Promise((resolve, reject) => rejectSubscribe = reject); this.ChannelManager.unsubscribe(this.rclient, "applied-ops", "1234567890abcdef"); diff --git a/services/real-time/test/unit/coffee/ConnectedUsersManagerTests.js b/services/real-time/test/unit/coffee/ConnectedUsersManagerTests.js index c1657e3669..f6b026fd1a 100644 --- a/services/real-time/test/unit/coffee/ConnectedUsersManagerTests.js +++ b/services/real-time/test/unit/coffee/ConnectedUsersManagerTests.js @@ -1,3 +1,11 @@ +/* eslint-disable + camelcase, + handle-callback-err, + no-return-assign, + no-unused-vars, +*/ +// TODO: This file was created by bulk-decaffeinate. +// Fix any style issues and re-enable lint. /* * decaffeinate suggestions: * DS102: Remove unnecessary code created because of implicit returns @@ -67,7 +75,7 @@ describe("ConnectedUsersManager", function() { }; return this.cursorData = { row: 12, column: 9, doc_id: '53c3b8c85fee64000023dc6e' };}); - afterEach(() => tk.reset()); + afterEach(function() { return tk.reset(); }); describe("updateUserPosition", function() { beforeEach(function() { diff --git a/services/real-time/test/unit/coffee/DocumentUpdaterControllerTests.js b/services/real-time/test/unit/coffee/DocumentUpdaterControllerTests.js index 9d15a77394..8b62b381a0 100644 --- a/services/real-time/test/unit/coffee/DocumentUpdaterControllerTests.js +++ b/services/real-time/test/unit/coffee/DocumentUpdaterControllerTests.js @@ -1,3 +1,9 @@ +/* eslint-disable + camelcase, + no-return-assign, +*/ +// TODO: This file was created by bulk-decaffeinate. +// Fix any style issues and re-enable lint. /* * decaffeinate suggestions: * DS101: Remove unnecessary use of Array.from diff --git a/services/real-time/test/unit/coffee/DocumentUpdaterManagerTests.js b/services/real-time/test/unit/coffee/DocumentUpdaterManagerTests.js index c5117a2fc0..49b08fa2b2 100644 --- a/services/real-time/test/unit/coffee/DocumentUpdaterManagerTests.js +++ b/services/real-time/test/unit/coffee/DocumentUpdaterManagerTests.js @@ -1,3 +1,10 @@ +/* eslint-disable + camelcase, + no-return-assign, + no-unused-vars, +*/ +// TODO: This file was created by bulk-decaffeinate. +// Fix any style issues and re-enable lint. /* * decaffeinate suggestions: * DS102: Remove unnecessary code created because of implicit returns diff --git a/services/real-time/test/unit/coffee/DrainManagerTests.js b/services/real-time/test/unit/coffee/DrainManagerTests.js index 87bdaeb6d3..6d6c8b826e 100644 --- a/services/real-time/test/unit/coffee/DrainManagerTests.js +++ b/services/real-time/test/unit/coffee/DrainManagerTests.js @@ -1,3 +1,9 @@ +/* eslint-disable + no-return-assign, + no-unused-vars, +*/ +// TODO: This file was created by bulk-decaffeinate. +// Fix any style issues and re-enable lint. /* * decaffeinate suggestions: * DS102: Remove unnecessary code created because of implicit returns diff --git a/services/real-time/test/unit/coffee/EventLoggerTests.js b/services/real-time/test/unit/coffee/EventLoggerTests.js index ab74861069..2d3b298e20 100644 --- a/services/real-time/test/unit/coffee/EventLoggerTests.js +++ b/services/real-time/test/unit/coffee/EventLoggerTests.js @@ -1,3 +1,8 @@ +/* eslint-disable + no-return-assign, +*/ +// TODO: This file was created by bulk-decaffeinate. +// Fix any style issues and re-enable lint. /* * decaffeinate suggestions: * DS102: Remove unnecessary code created because of implicit returns @@ -28,7 +33,7 @@ describe('EventLogger', function() { return this.message_2 = "message-2"; }); - afterEach(() => tk.reset()); + afterEach(function() { return tk.reset(); }); return describe('checkEventOrder', function() { @@ -81,7 +86,7 @@ describe('EventLogger', function() { }); }); - return describe('after MAX_STALE_TIME_IN_MS', () => it('should flush old entries', function() { + return describe('after MAX_STALE_TIME_IN_MS', function() { return it('should flush old entries', function() { let status; this.EventLogger.MAX_EVENTS_BEFORE_CLEAN = 10; this.EventLogger.checkEventOrder(this.channel, this.id_1, this.message_1); @@ -96,6 +101,6 @@ describe('EventLogger', function() { this.EventLogger.checkEventOrder(this.channel, 'other-1', this.message_2); status = this.EventLogger.checkEventOrder(this.channel, this.id_1, this.message_1); return expect(status).to.be.undefined; - })); + }); }); }); }); \ No newline at end of file diff --git a/services/real-time/test/unit/coffee/RoomManagerTests.js b/services/real-time/test/unit/coffee/RoomManagerTests.js index 63c25b3eae..b356d0ee5e 100644 --- a/services/real-time/test/unit/coffee/RoomManagerTests.js +++ b/services/real-time/test/unit/coffee/RoomManagerTests.js @@ -1,3 +1,10 @@ +/* eslint-disable + no-return-assign, + no-unused-vars, + promise/param-names, +*/ +// TODO: This file was created by bulk-decaffeinate. +// Fix any style issues and re-enable lint. /* * decaffeinate suggestions: * DS102: Remove unnecessary code created because of implicit returns @@ -31,7 +38,7 @@ describe('RoomManager', function() { return sinon.spy(this.RoomEvents, 'once'); }); - describe("emitOnCompletion", () => describe("when a subscribe errors", function() { + describe("emitOnCompletion", function() { return describe("when a subscribe errors", function() { afterEach(function() { return process.removeListener("unhandledRejection", this.onUnhandled); }); @@ -43,7 +50,7 @@ describe('RoomManager', function() { }; process.on("unhandledRejection", this.onUnhandled); - let reject = undefined; + let reject; const subscribePromise = new Promise((_, r) => reject = r); const promises = [subscribePromise]; const eventName = "project-subscribed-123"; @@ -55,7 +62,7 @@ describe('RoomManager', function() { return it("should keep going", function() { return expect(this.unhandledError).to.not.exist; }); - })); + }); }); describe("joinProject", function() { @@ -242,7 +249,7 @@ describe('RoomManager', function() { }); - return describe("leaveProjectAndDocs", () => describe("when the client is connected to the project and multiple docs", function() { + return describe("leaveProjectAndDocs", function() { return describe("when the client is connected to the project and multiple docs", function() { beforeEach(function() { this.RoomManager._roomsClientIsIn = sinon.stub().returns([this.project_id, this.doc_id, this.other_doc_id]); @@ -355,5 +362,5 @@ describe('RoomManager', function() { return this.RoomEvents.emit.called.should.equal(false); }); }); - })); + }); }); }); \ No newline at end of file diff --git a/services/real-time/test/unit/coffee/SafeJsonParseTest.js b/services/real-time/test/unit/coffee/SafeJsonParseTest.js index f417513e47..58fed31397 100644 --- a/services/real-time/test/unit/coffee/SafeJsonParseTest.js +++ b/services/real-time/test/unit/coffee/SafeJsonParseTest.js @@ -1,3 +1,11 @@ +/* eslint-disable + camelcase, + handle-callback-err, + no-return-assign, + no-useless-escape, +*/ +// TODO: This file was created by bulk-decaffeinate. +// Fix any style issues and re-enable lint. /* * decaffeinate suggestions: * DS102: Remove unnecessary code created because of implicit returns @@ -23,14 +31,14 @@ describe('SafeJsonParse', function() { return describe("parse", function() { it("should parse documents correctly", function(done) { - return this.SafeJsonParse.parse('{"foo": "bar"}', function(error, parsed) { + return this.SafeJsonParse.parse('{"foo": "bar"}', (error, parsed) => { expect(parsed).to.deep.equal({foo: "bar"}); return done(); }); }); it("should return an error on bad data", function(done) { - return this.SafeJsonParse.parse('blah', function(error, parsed) { + return this.SafeJsonParse.parse('blah', (error, parsed) => { expect(error).to.exist; return done(); }); diff --git a/services/real-time/test/unit/coffee/SessionSocketsTests.js b/services/real-time/test/unit/coffee/SessionSocketsTests.js index d85be502a7..a57a58bfac 100644 --- a/services/real-time/test/unit/coffee/SessionSocketsTests.js +++ b/services/real-time/test/unit/coffee/SessionSocketsTests.js @@ -1,3 +1,9 @@ +/* eslint-disable + handle-callback-err, + no-return-assign, +*/ +// TODO: This file was created by bulk-decaffeinate. +// Fix any style issues and re-enable lint. /* * decaffeinate suggestions: * DS102: Remove unnecessary code created because of implicit returns @@ -41,7 +47,7 @@ describe('SessionSockets', function() { return this.socket = {handshake: {}};}); it('should return a lookup error', function(done) { - return this.checkSocket(this.socket, function(error) { + return this.checkSocket(this.socket, (error) => { expect(error).to.exist; expect(error.message).to.equal('could not look up session by key'); return done(); @@ -61,7 +67,7 @@ describe('SessionSockets', function() { return this.socket = {handshake: {_signedCookies: {other: 1}}};}); it('should return a lookup error', function(done) { - return this.checkSocket(this.socket, function(error) { + return this.checkSocket(this.socket, (error) => { expect(error).to.exist; expect(error.message).to.equal('could not look up session by key'); return done(); @@ -88,7 +94,7 @@ describe('SessionSockets', function() { }); return it('should return a redis error', function(done) { - return this.checkSocket(this.socket, function(error) { + return this.checkSocket(this.socket, (error) => { expect(error).to.exist; expect(error.message).to.equal('Redis: something went wrong'); return done(); @@ -108,7 +114,7 @@ describe('SessionSockets', function() { }); return it('should return a lookup error', function(done) { - return this.checkSocket(this.socket, function(error) { + return this.checkSocket(this.socket, (error) => { expect(error).to.exist; expect(error.message).to.equal('could not look up session by key'); return done(); @@ -128,14 +134,14 @@ describe('SessionSockets', function() { }); it('should not return an error', function(done) { - return this.checkSocket(this.socket, function(error) { + return this.checkSocket(this.socket, (error) => { expect(error).to.not.exist; return done(); }); }); return it('should return the session', function(done) { - return this.checkSocket(this.socket, function(error, s, session) { + return this.checkSocket(this.socket, (error, s, session) => { expect(session).to.deep.equal({user: {_id: '123'}}); return done(); }); @@ -154,14 +160,14 @@ describe('SessionSockets', function() { }); it('should not return an error', function(done) { - return this.checkSocket(this.socket, function(error) { + return this.checkSocket(this.socket, (error) => { expect(error).to.not.exist; return done(); }); }); return it('should return the other session', function(done) { - return this.checkSocket(this.socket, function(error, s, session) { + return this.checkSocket(this.socket, (error, s, session) => { expect(session).to.deep.equal({user: {_id: 'abc'}}); return done(); }); diff --git a/services/real-time/test/unit/coffee/WebApiManagerTests.js b/services/real-time/test/unit/coffee/WebApiManagerTests.js index 19d2bbe444..c868cbaf0e 100644 --- a/services/real-time/test/unit/coffee/WebApiManagerTests.js +++ b/services/real-time/test/unit/coffee/WebApiManagerTests.js @@ -1,3 +1,9 @@ +/* eslint-disable + no-return-assign, + no-unused-vars, +*/ +// TODO: This file was created by bulk-decaffeinate. +// Fix any style issues and re-enable lint. /* * decaffeinate suggestions: * DS102: Remove unnecessary code created because of implicit returns diff --git a/services/real-time/test/unit/coffee/WebsocketControllerTests.js b/services/real-time/test/unit/coffee/WebsocketControllerTests.js index 92d64d7cd2..58f417d1ca 100644 --- a/services/real-time/test/unit/coffee/WebsocketControllerTests.js +++ b/services/real-time/test/unit/coffee/WebsocketControllerTests.js @@ -1,3 +1,11 @@ +/* eslint-disable + camelcase, + no-return-assign, + no-throw-literal, + no-unused-vars, +*/ +// TODO: This file was created by bulk-decaffeinate. +// Fix any style issues and re-enable lint. /* * decaffeinate suggestions: * DS101: Remove unnecessary use of Array.from @@ -50,7 +58,7 @@ describe('WebsocketController', function() { } });}); - afterEach(() => tk.reset()); + afterEach(function() { return tk.reset(); }); describe("joinProject", function() { describe("when authorised", function() { @@ -81,37 +89,37 @@ describe('WebsocketController', function() { }); it("should set the privilege level on the client", function() { - return this.client.ol_context["privilege_level"].should.equal(this.privilegeLevel); + return this.client.ol_context.privilege_level.should.equal(this.privilegeLevel); }); it("should set the user's id on the client", function() { - return this.client.ol_context["user_id"].should.equal(this.user._id); + return this.client.ol_context.user_id.should.equal(this.user._id); }); it("should set the user's email on the client", function() { - return this.client.ol_context["email"].should.equal(this.user.email); + return this.client.ol_context.email.should.equal(this.user.email); }); it("should set the user's first_name on the client", function() { - return this.client.ol_context["first_name"].should.equal(this.user.first_name); + return this.client.ol_context.first_name.should.equal(this.user.first_name); }); it("should set the user's last_name on the client", function() { - return this.client.ol_context["last_name"].should.equal(this.user.last_name); + return this.client.ol_context.last_name.should.equal(this.user.last_name); }); it("should set the user's sign up date on the client", function() { - return this.client.ol_context["signup_date"].should.equal(this.user.signUpDate); + return this.client.ol_context.signup_date.should.equal(this.user.signUpDate); }); it("should set the user's login_count on the client", function() { - return this.client.ol_context["login_count"].should.equal(this.user.loginCount); + return this.client.ol_context.login_count.should.equal(this.user.loginCount); }); it("should set the connected time on the client", function() { - return this.client.ol_context["connected_time"].should.equal(new Date()); + return this.client.ol_context.connected_time.should.equal(new Date()); }); it("should set the project_id on the client", function() { - return this.client.ol_context["project_id"].should.equal(this.project_id); + return this.client.ol_context.project_id.should.equal(this.project_id); }); it("should set the project owner id on the client", function() { - return this.client.ol_context["owner_id"].should.equal(this.owner_id); + return this.client.ol_context.owner_id.should.equal(this.owner_id); }); it("should set the is_restricted_user flag on the client", function() { - return this.client.ol_context["is_restricted_user"].should.equal(this.isRestrictedUser); + return this.client.ol_context.is_restricted_user.should.equal(this.isRestrictedUser); }); it("should call the callback with the project, privilegeLevel and protocolVersion", function() { return this.callback @@ -1010,7 +1018,7 @@ describe('WebsocketController', function() { }); describe("after 100ms", function() { - beforeEach(done => setTimeout(done, 100)); + beforeEach(function(done) { return setTimeout(done, 100); }); it("should send an otUpdateError the client", function() { return this.client.emit.calledWith('otUpdateError').should.equal(true); @@ -1050,39 +1058,39 @@ describe('WebsocketController', function() { return this.AuthorizationManager.assertClientCanViewProjectAndDoc = sinon.stub(); }); - describe("with a read-write client", () => it("should return successfully", function(done) { + describe("with a read-write client", function() { return it("should return successfully", function(done) { this.AuthorizationManager.assertClientCanEditProjectAndDoc.yields(null); - return this.WebsocketController._assertClientCanApplyUpdate(this.client, this.doc_id, this.edit_update, function(error) { + return this.WebsocketController._assertClientCanApplyUpdate(this.client, this.doc_id, this.edit_update, (error) => { expect(error).to.be.null; return done(); }); - })); + }); }); - describe("with a read-only client and an edit op", () => it("should return an error", function(done) { + describe("with a read-only client and an edit op", function() { return it("should return an error", function(done) { this.AuthorizationManager.assertClientCanEditProjectAndDoc.yields(new Error("not authorized")); this.AuthorizationManager.assertClientCanViewProjectAndDoc.yields(null); - return this.WebsocketController._assertClientCanApplyUpdate(this.client, this.doc_id, this.edit_update, function(error) { + return this.WebsocketController._assertClientCanApplyUpdate(this.client, this.doc_id, this.edit_update, (error) => { expect(error.message).to.equal("not authorized"); return done(); }); - })); + }); }); - describe("with a read-only client and a comment op", () => it("should return successfully", function(done) { + describe("with a read-only client and a comment op", function() { return it("should return successfully", function(done) { this.AuthorizationManager.assertClientCanEditProjectAndDoc.yields(new Error("not authorized")); this.AuthorizationManager.assertClientCanViewProjectAndDoc.yields(null); - return this.WebsocketController._assertClientCanApplyUpdate(this.client, this.doc_id, this.comment_update, function(error) { + return this.WebsocketController._assertClientCanApplyUpdate(this.client, this.doc_id, this.comment_update, (error) => { expect(error).to.be.null; return done(); }); - })); + }); }); - return describe("with a totally unauthorized client", () => it("should return an error", function(done) { + return describe("with a totally unauthorized client", function() { return it("should return an error", function(done) { this.AuthorizationManager.assertClientCanEditProjectAndDoc.yields(new Error("not authorized")); this.AuthorizationManager.assertClientCanViewProjectAndDoc.yields(new Error("not authorized")); - return this.WebsocketController._assertClientCanApplyUpdate(this.client, this.doc_id, this.comment_update, function(error) { + return this.WebsocketController._assertClientCanApplyUpdate(this.client, this.doc_id, this.comment_update, (error) => { expect(error.message).to.equal("not authorized"); return done(); }); - })); + }); }); }); }); diff --git a/services/real-time/test/unit/coffee/WebsocketLoadBalancerTests.js b/services/real-time/test/unit/coffee/WebsocketLoadBalancerTests.js index 5b9ab079fb..0d0c0f6b9d 100644 --- a/services/real-time/test/unit/coffee/WebsocketLoadBalancerTests.js +++ b/services/real-time/test/unit/coffee/WebsocketLoadBalancerTests.js @@ -1,3 +1,8 @@ +/* eslint-disable + no-return-assign, +*/ +// TODO: This file was created by bulk-decaffeinate. +// Fix any style issues and re-enable lint. /* * decaffeinate suggestions: * DS101: Remove unnecessary use of Array.from diff --git a/services/real-time/test/unit/coffee/helpers/MockClient.js b/services/real-time/test/unit/coffee/helpers/MockClient.js index bdf711ac8d..5f9b019db4 100644 --- a/services/real-time/test/unit/coffee/helpers/MockClient.js +++ b/services/real-time/test/unit/coffee/helpers/MockClient.js @@ -1,3 +1,8 @@ +/* eslint-disable + no-unused-vars, +*/ +// TODO: This file was created by bulk-decaffeinate. +// Fix any style issues and re-enable lint. let MockClient; const sinon = require('sinon'); @@ -12,5 +17,6 @@ module.exports = (MockClient = class MockClient { this.id = idCounter++; this.publicId = idCounter++; } + disconnect() {} }); From 68e2adebf52abe0bb6869dd006f6e8575de9cffb Mon Sep 17 00:00:00 2001 From: decaffeinate Date: Tue, 23 Jun 2020 18:30:06 +0100 Subject: [PATCH 376/491] decaffeinate: rename test/unit/coffee to test/unit/js --- .../test/unit/{coffee => js}/AuthorizationManagerTests.js | 0 .../real-time/test/unit/{coffee => js}/ChannelManagerTests.js | 0 .../test/unit/{coffee => js}/ConnectedUsersManagerTests.js | 0 .../test/unit/{coffee => js}/DocumentUpdaterControllerTests.js | 0 .../test/unit/{coffee => js}/DocumentUpdaterManagerTests.js | 0 services/real-time/test/unit/{coffee => js}/DrainManagerTests.js | 0 services/real-time/test/unit/{coffee => js}/EventLoggerTests.js | 0 services/real-time/test/unit/{coffee => js}/RoomManagerTests.js | 0 services/real-time/test/unit/{coffee => js}/SafeJsonParseTest.js | 0 .../real-time/test/unit/{coffee => js}/SessionSocketsTests.js | 0 services/real-time/test/unit/{coffee => js}/WebApiManagerTests.js | 0 .../test/unit/{coffee => js}/WebsocketControllerTests.js | 0 .../test/unit/{coffee => js}/WebsocketLoadBalancerTests.js | 0 services/real-time/test/unit/{coffee => js}/helpers/MockClient.js | 0 14 files changed, 0 insertions(+), 0 deletions(-) rename services/real-time/test/unit/{coffee => js}/AuthorizationManagerTests.js (100%) rename services/real-time/test/unit/{coffee => js}/ChannelManagerTests.js (100%) rename services/real-time/test/unit/{coffee => js}/ConnectedUsersManagerTests.js (100%) rename services/real-time/test/unit/{coffee => js}/DocumentUpdaterControllerTests.js (100%) rename services/real-time/test/unit/{coffee => js}/DocumentUpdaterManagerTests.js (100%) rename services/real-time/test/unit/{coffee => js}/DrainManagerTests.js (100%) rename services/real-time/test/unit/{coffee => js}/EventLoggerTests.js (100%) rename services/real-time/test/unit/{coffee => js}/RoomManagerTests.js (100%) rename services/real-time/test/unit/{coffee => js}/SafeJsonParseTest.js (100%) rename services/real-time/test/unit/{coffee => js}/SessionSocketsTests.js (100%) rename services/real-time/test/unit/{coffee => js}/WebApiManagerTests.js (100%) rename services/real-time/test/unit/{coffee => js}/WebsocketControllerTests.js (100%) rename services/real-time/test/unit/{coffee => js}/WebsocketLoadBalancerTests.js (100%) rename services/real-time/test/unit/{coffee => js}/helpers/MockClient.js (100%) diff --git a/services/real-time/test/unit/coffee/AuthorizationManagerTests.js b/services/real-time/test/unit/js/AuthorizationManagerTests.js similarity index 100% rename from services/real-time/test/unit/coffee/AuthorizationManagerTests.js rename to services/real-time/test/unit/js/AuthorizationManagerTests.js diff --git a/services/real-time/test/unit/coffee/ChannelManagerTests.js b/services/real-time/test/unit/js/ChannelManagerTests.js similarity index 100% rename from services/real-time/test/unit/coffee/ChannelManagerTests.js rename to services/real-time/test/unit/js/ChannelManagerTests.js diff --git a/services/real-time/test/unit/coffee/ConnectedUsersManagerTests.js b/services/real-time/test/unit/js/ConnectedUsersManagerTests.js similarity index 100% rename from services/real-time/test/unit/coffee/ConnectedUsersManagerTests.js rename to services/real-time/test/unit/js/ConnectedUsersManagerTests.js diff --git a/services/real-time/test/unit/coffee/DocumentUpdaterControllerTests.js b/services/real-time/test/unit/js/DocumentUpdaterControllerTests.js similarity index 100% rename from services/real-time/test/unit/coffee/DocumentUpdaterControllerTests.js rename to services/real-time/test/unit/js/DocumentUpdaterControllerTests.js diff --git a/services/real-time/test/unit/coffee/DocumentUpdaterManagerTests.js b/services/real-time/test/unit/js/DocumentUpdaterManagerTests.js similarity index 100% rename from services/real-time/test/unit/coffee/DocumentUpdaterManagerTests.js rename to services/real-time/test/unit/js/DocumentUpdaterManagerTests.js diff --git a/services/real-time/test/unit/coffee/DrainManagerTests.js b/services/real-time/test/unit/js/DrainManagerTests.js similarity index 100% rename from services/real-time/test/unit/coffee/DrainManagerTests.js rename to services/real-time/test/unit/js/DrainManagerTests.js diff --git a/services/real-time/test/unit/coffee/EventLoggerTests.js b/services/real-time/test/unit/js/EventLoggerTests.js similarity index 100% rename from services/real-time/test/unit/coffee/EventLoggerTests.js rename to services/real-time/test/unit/js/EventLoggerTests.js diff --git a/services/real-time/test/unit/coffee/RoomManagerTests.js b/services/real-time/test/unit/js/RoomManagerTests.js similarity index 100% rename from services/real-time/test/unit/coffee/RoomManagerTests.js rename to services/real-time/test/unit/js/RoomManagerTests.js diff --git a/services/real-time/test/unit/coffee/SafeJsonParseTest.js b/services/real-time/test/unit/js/SafeJsonParseTest.js similarity index 100% rename from services/real-time/test/unit/coffee/SafeJsonParseTest.js rename to services/real-time/test/unit/js/SafeJsonParseTest.js diff --git a/services/real-time/test/unit/coffee/SessionSocketsTests.js b/services/real-time/test/unit/js/SessionSocketsTests.js similarity index 100% rename from services/real-time/test/unit/coffee/SessionSocketsTests.js rename to services/real-time/test/unit/js/SessionSocketsTests.js diff --git a/services/real-time/test/unit/coffee/WebApiManagerTests.js b/services/real-time/test/unit/js/WebApiManagerTests.js similarity index 100% rename from services/real-time/test/unit/coffee/WebApiManagerTests.js rename to services/real-time/test/unit/js/WebApiManagerTests.js diff --git a/services/real-time/test/unit/coffee/WebsocketControllerTests.js b/services/real-time/test/unit/js/WebsocketControllerTests.js similarity index 100% rename from services/real-time/test/unit/coffee/WebsocketControllerTests.js rename to services/real-time/test/unit/js/WebsocketControllerTests.js diff --git a/services/real-time/test/unit/coffee/WebsocketLoadBalancerTests.js b/services/real-time/test/unit/js/WebsocketLoadBalancerTests.js similarity index 100% rename from services/real-time/test/unit/coffee/WebsocketLoadBalancerTests.js rename to services/real-time/test/unit/js/WebsocketLoadBalancerTests.js diff --git a/services/real-time/test/unit/coffee/helpers/MockClient.js b/services/real-time/test/unit/js/helpers/MockClient.js similarity index 100% rename from services/real-time/test/unit/coffee/helpers/MockClient.js rename to services/real-time/test/unit/js/helpers/MockClient.js From 3eceb8a5f6665bc6a8641bba54884606f6508b66 Mon Sep 17 00:00:00 2001 From: decaffeinate Date: Tue, 23 Jun 2020 18:30:16 +0100 Subject: [PATCH 377/491] prettier: convert test/unit decaffeinated files to Prettier format --- .../test/unit/js/AuthorizationManagerTests.js | 454 +-- .../test/unit/js/ChannelManagerTests.js | 622 ++-- .../unit/js/ConnectedUsersManagerTests.js | 566 ++-- .../unit/js/DocumentUpdaterControllerTests.js | 408 +-- .../unit/js/DocumentUpdaterManagerTests.js | 556 ++-- .../test/unit/js/DrainManagerTests.js | 217 +- .../test/unit/js/EventLoggerTests.js | 219 +- .../test/unit/js/RoomManagerTests.js | 685 +++-- .../test/unit/js/SafeJsonParseTest.js | 88 +- .../test/unit/js/SessionSocketsTests.js | 307 +- .../test/unit/js/WebApiManagerTests.js | 237 +- .../test/unit/js/WebsocketControllerTests.js | 2562 ++++++++++------- .../unit/js/WebsocketLoadBalancerTests.js | 458 +-- .../test/unit/js/helpers/MockClient.js | 28 +- 14 files changed, 4344 insertions(+), 3063 deletions(-) diff --git a/services/real-time/test/unit/js/AuthorizationManagerTests.js b/services/real-time/test/unit/js/AuthorizationManagerTests.js index d3aa6be9fa..3093017a39 100644 --- a/services/real-time/test/unit/js/AuthorizationManagerTests.js +++ b/services/real-time/test/unit/js/AuthorizationManagerTests.js @@ -9,214 +9,312 @@ * DS102: Remove unnecessary code created because of implicit returns * Full docs: https://github.com/decaffeinate/decaffeinate/blob/master/docs/suggestions.md */ -const chai = require("chai"); -chai.should(); -const { - expect -} = chai; -const sinon = require("sinon"); -const SandboxedModule = require('sandboxed-module'); -const path = require("path"); -const modulePath = '../../../app/js/AuthorizationManager'; +const chai = require('chai') +chai.should() +const { expect } = chai +const sinon = require('sinon') +const SandboxedModule = require('sandboxed-module') +const path = require('path') +const modulePath = '../../../app/js/AuthorizationManager' -describe('AuthorizationManager', function() { - beforeEach(function() { - this.client = - {ol_context: {}}; +describe('AuthorizationManager', function () { + beforeEach(function () { + this.client = { ol_context: {} } - return this.AuthorizationManager = SandboxedModule.require(modulePath, {requires: {}});}); + return (this.AuthorizationManager = SandboxedModule.require(modulePath, { + requires: {} + })) + }) - describe("assertClientCanViewProject", function() { - it("should allow the readOnly privilegeLevel", function(done) { - this.client.ol_context.privilege_level = "readOnly"; - return this.AuthorizationManager.assertClientCanViewProject(this.client, (error) => { - expect(error).to.be.null; - return done(); - }); - }); + describe('assertClientCanViewProject', function () { + it('should allow the readOnly privilegeLevel', function (done) { + this.client.ol_context.privilege_level = 'readOnly' + return this.AuthorizationManager.assertClientCanViewProject( + this.client, + (error) => { + expect(error).to.be.null + return done() + } + ) + }) - it("should allow the readAndWrite privilegeLevel", function(done) { - this.client.ol_context.privilege_level = "readAndWrite"; - return this.AuthorizationManager.assertClientCanViewProject(this.client, (error) => { - expect(error).to.be.null; - return done(); - }); - }); + it('should allow the readAndWrite privilegeLevel', function (done) { + this.client.ol_context.privilege_level = 'readAndWrite' + return this.AuthorizationManager.assertClientCanViewProject( + this.client, + (error) => { + expect(error).to.be.null + return done() + } + ) + }) - it("should allow the owner privilegeLevel", function(done) { - this.client.ol_context.privilege_level = "owner"; - return this.AuthorizationManager.assertClientCanViewProject(this.client, (error) => { - expect(error).to.be.null; - return done(); - }); - }); + it('should allow the owner privilegeLevel', function (done) { + this.client.ol_context.privilege_level = 'owner' + return this.AuthorizationManager.assertClientCanViewProject( + this.client, + (error) => { + expect(error).to.be.null + return done() + } + ) + }) - return it("should return an error with any other privilegeLevel", function(done) { - this.client.ol_context.privilege_level = "unknown"; - return this.AuthorizationManager.assertClientCanViewProject(this.client, (error) => { - error.message.should.equal("not authorized"); - return done(); - }); - }); - }); + return it('should return an error with any other privilegeLevel', function (done) { + this.client.ol_context.privilege_level = 'unknown' + return this.AuthorizationManager.assertClientCanViewProject( + this.client, + (error) => { + error.message.should.equal('not authorized') + return done() + } + ) + }) + }) - describe("assertClientCanEditProject", function() { - it("should not allow the readOnly privilegeLevel", function(done) { - this.client.ol_context.privilege_level = "readOnly"; - return this.AuthorizationManager.assertClientCanEditProject(this.client, (error) => { - error.message.should.equal("not authorized"); - return done(); - }); - }); + describe('assertClientCanEditProject', function () { + it('should not allow the readOnly privilegeLevel', function (done) { + this.client.ol_context.privilege_level = 'readOnly' + return this.AuthorizationManager.assertClientCanEditProject( + this.client, + (error) => { + error.message.should.equal('not authorized') + return done() + } + ) + }) - it("should allow the readAndWrite privilegeLevel", function(done) { - this.client.ol_context.privilege_level = "readAndWrite"; - return this.AuthorizationManager.assertClientCanEditProject(this.client, (error) => { - expect(error).to.be.null; - return done(); - }); - }); + it('should allow the readAndWrite privilegeLevel', function (done) { + this.client.ol_context.privilege_level = 'readAndWrite' + return this.AuthorizationManager.assertClientCanEditProject( + this.client, + (error) => { + expect(error).to.be.null + return done() + } + ) + }) - it("should allow the owner privilegeLevel", function(done) { - this.client.ol_context.privilege_level = "owner"; - return this.AuthorizationManager.assertClientCanEditProject(this.client, (error) => { - expect(error).to.be.null; - return done(); - }); - }); + it('should allow the owner privilegeLevel', function (done) { + this.client.ol_context.privilege_level = 'owner' + return this.AuthorizationManager.assertClientCanEditProject( + this.client, + (error) => { + expect(error).to.be.null + return done() + } + ) + }) - return it("should return an error with any other privilegeLevel", function(done) { - this.client.ol_context.privilege_level = "unknown"; - return this.AuthorizationManager.assertClientCanEditProject(this.client, (error) => { - error.message.should.equal("not authorized"); - return done(); - }); - }); - }); + return it('should return an error with any other privilegeLevel', function (done) { + this.client.ol_context.privilege_level = 'unknown' + return this.AuthorizationManager.assertClientCanEditProject( + this.client, + (error) => { + error.message.should.equal('not authorized') + return done() + } + ) + }) + }) - // check doc access for project + // check doc access for project - describe("assertClientCanViewProjectAndDoc", function() { - beforeEach(function() { - this.doc_id = "12345"; - this.callback = sinon.stub(); - return this.client.ol_context = {};}); + describe('assertClientCanViewProjectAndDoc', function () { + beforeEach(function () { + this.doc_id = '12345' + this.callback = sinon.stub() + return (this.client.ol_context = {}) + }) - describe("when not authorised at the project level", function() { - beforeEach(function() { - return this.client.ol_context.privilege_level = "unknown"; - }); + describe('when not authorised at the project level', function () { + beforeEach(function () { + return (this.client.ol_context.privilege_level = 'unknown') + }) - it("should not allow access", function() { - return this.AuthorizationManager.assertClientCanViewProjectAndDoc(this.client, this.doc_id, err => err.message.should.equal("not authorized")); - }); + it('should not allow access', function () { + return this.AuthorizationManager.assertClientCanViewProjectAndDoc( + this.client, + this.doc_id, + (err) => err.message.should.equal('not authorized') + ) + }) - return describe("even when authorised at the doc level", function() { - beforeEach(function(done) { - return this.AuthorizationManager.addAccessToDoc(this.client, this.doc_id, done); - }); + return describe('even when authorised at the doc level', function () { + beforeEach(function (done) { + return this.AuthorizationManager.addAccessToDoc( + this.client, + this.doc_id, + done + ) + }) - return it("should not allow access", function() { - return this.AuthorizationManager.assertClientCanViewProjectAndDoc(this.client, this.doc_id, err => err.message.should.equal("not authorized")); - }); - }); - }); + return it('should not allow access', function () { + return this.AuthorizationManager.assertClientCanViewProjectAndDoc( + this.client, + this.doc_id, + (err) => err.message.should.equal('not authorized') + ) + }) + }) + }) - return describe("when authorised at the project level", function() { - beforeEach(function() { - return this.client.ol_context.privilege_level = "readOnly"; - }); + return describe('when authorised at the project level', function () { + beforeEach(function () { + return (this.client.ol_context.privilege_level = 'readOnly') + }) - describe("and not authorised at the document level", function() { return it("should not allow access", function() { - return this.AuthorizationManager.assertClientCanViewProjectAndDoc(this.client, this.doc_id, err => err.message.should.equal("not authorized")); - }); }); + describe('and not authorised at the document level', function () { + return it('should not allow access', function () { + return this.AuthorizationManager.assertClientCanViewProjectAndDoc( + this.client, + this.doc_id, + (err) => err.message.should.equal('not authorized') + ) + }) + }) - describe("and authorised at the document level", function() { - beforeEach(function(done) { - return this.AuthorizationManager.addAccessToDoc(this.client, this.doc_id, done); - }); + describe('and authorised at the document level', function () { + beforeEach(function (done) { + return this.AuthorizationManager.addAccessToDoc( + this.client, + this.doc_id, + done + ) + }) - return it("should allow access", function() { - this.AuthorizationManager.assertClientCanViewProjectAndDoc(this.client, this.doc_id, this.callback); - return this.callback - .calledWith(null) - .should.equal(true); - }); - }); + return it('should allow access', function () { + this.AuthorizationManager.assertClientCanViewProjectAndDoc( + this.client, + this.doc_id, + this.callback + ) + return this.callback.calledWith(null).should.equal(true) + }) + }) - return describe("when document authorisation is added and then removed", function() { - beforeEach(function(done) { - return this.AuthorizationManager.addAccessToDoc(this.client, this.doc_id, () => { - return this.AuthorizationManager.removeAccessToDoc(this.client, this.doc_id, done); - }); - }); + return describe('when document authorisation is added and then removed', function () { + beforeEach(function (done) { + return this.AuthorizationManager.addAccessToDoc( + this.client, + this.doc_id, + () => { + return this.AuthorizationManager.removeAccessToDoc( + this.client, + this.doc_id, + done + ) + } + ) + }) - return it("should deny access", function() { - return this.AuthorizationManager.assertClientCanViewProjectAndDoc(this.client, this.doc_id, err => err.message.should.equal("not authorized")); - }); - }); - }); - }); + return it('should deny access', function () { + return this.AuthorizationManager.assertClientCanViewProjectAndDoc( + this.client, + this.doc_id, + (err) => err.message.should.equal('not authorized') + ) + }) + }) + }) + }) - return describe("assertClientCanEditProjectAndDoc", function() { - beforeEach(function() { - this.doc_id = "12345"; - this.callback = sinon.stub(); - return this.client.ol_context = {};}); + return describe('assertClientCanEditProjectAndDoc', function () { + beforeEach(function () { + this.doc_id = '12345' + this.callback = sinon.stub() + return (this.client.ol_context = {}) + }) - describe("when not authorised at the project level", function() { - beforeEach(function() { - return this.client.ol_context.privilege_level = "readOnly"; - }); + describe('when not authorised at the project level', function () { + beforeEach(function () { + return (this.client.ol_context.privilege_level = 'readOnly') + }) - it("should not allow access", function() { - return this.AuthorizationManager.assertClientCanEditProjectAndDoc(this.client, this.doc_id, err => err.message.should.equal("not authorized")); - }); + it('should not allow access', function () { + return this.AuthorizationManager.assertClientCanEditProjectAndDoc( + this.client, + this.doc_id, + (err) => err.message.should.equal('not authorized') + ) + }) - return describe("even when authorised at the doc level", function() { - beforeEach(function(done) { - return this.AuthorizationManager.addAccessToDoc(this.client, this.doc_id, done); - }); + return describe('even when authorised at the doc level', function () { + beforeEach(function (done) { + return this.AuthorizationManager.addAccessToDoc( + this.client, + this.doc_id, + done + ) + }) - return it("should not allow access", function() { - return this.AuthorizationManager.assertClientCanEditProjectAndDoc(this.client, this.doc_id, err => err.message.should.equal("not authorized")); - }); - }); - }); + return it('should not allow access', function () { + return this.AuthorizationManager.assertClientCanEditProjectAndDoc( + this.client, + this.doc_id, + (err) => err.message.should.equal('not authorized') + ) + }) + }) + }) - return describe("when authorised at the project level", function() { - beforeEach(function() { - return this.client.ol_context.privilege_level = "readAndWrite"; - }); + return describe('when authorised at the project level', function () { + beforeEach(function () { + return (this.client.ol_context.privilege_level = 'readAndWrite') + }) - describe("and not authorised at the document level", function() { return it("should not allow access", function() { - return this.AuthorizationManager.assertClientCanEditProjectAndDoc(this.client, this.doc_id, err => err.message.should.equal("not authorized")); - }); }); + describe('and not authorised at the document level', function () { + return it('should not allow access', function () { + return this.AuthorizationManager.assertClientCanEditProjectAndDoc( + this.client, + this.doc_id, + (err) => err.message.should.equal('not authorized') + ) + }) + }) - describe("and authorised at the document level", function() { - beforeEach(function(done) { - return this.AuthorizationManager.addAccessToDoc(this.client, this.doc_id, done); - }); + describe('and authorised at the document level', function () { + beforeEach(function (done) { + return this.AuthorizationManager.addAccessToDoc( + this.client, + this.doc_id, + done + ) + }) - return it("should allow access", function() { - this.AuthorizationManager.assertClientCanEditProjectAndDoc(this.client, this.doc_id, this.callback); - return this.callback - .calledWith(null) - .should.equal(true); - }); - }); + return it('should allow access', function () { + this.AuthorizationManager.assertClientCanEditProjectAndDoc( + this.client, + this.doc_id, + this.callback + ) + return this.callback.calledWith(null).should.equal(true) + }) + }) - return describe("when document authorisation is added and then removed", function() { - beforeEach(function(done) { - return this.AuthorizationManager.addAccessToDoc(this.client, this.doc_id, () => { - return this.AuthorizationManager.removeAccessToDoc(this.client, this.doc_id, done); - }); - }); + return describe('when document authorisation is added and then removed', function () { + beforeEach(function (done) { + return this.AuthorizationManager.addAccessToDoc( + this.client, + this.doc_id, + () => { + return this.AuthorizationManager.removeAccessToDoc( + this.client, + this.doc_id, + done + ) + } + ) + }) - return it("should deny access", function() { - return this.AuthorizationManager.assertClientCanEditProjectAndDoc(this.client, this.doc_id, err => err.message.should.equal("not authorized")); - }); - }); - }); - }); -}); + return it('should deny access', function () { + return this.AuthorizationManager.assertClientCanEditProjectAndDoc( + this.client, + this.doc_id, + (err) => err.message.should.equal('not authorized') + ) + }) + }) + }) + }) +}) diff --git a/services/real-time/test/unit/js/ChannelManagerTests.js b/services/real-time/test/unit/js/ChannelManagerTests.js index 1b71565975..6026f6ab5c 100644 --- a/services/real-time/test/unit/js/ChannelManagerTests.js +++ b/services/real-time/test/unit/js/ChannelManagerTests.js @@ -9,272 +9,430 @@ * DS102: Remove unnecessary code created because of implicit returns * Full docs: https://github.com/decaffeinate/decaffeinate/blob/master/docs/suggestions.md */ -const chai = require('chai'); -const should = chai.should(); -const { - expect -} = chai; -const sinon = require("sinon"); -const modulePath = "../../../app/js/ChannelManager.js"; -const SandboxedModule = require('sandboxed-module'); +const chai = require('chai') +const should = chai.should() +const { expect } = chai +const sinon = require('sinon') +const modulePath = '../../../app/js/ChannelManager.js' +const SandboxedModule = require('sandboxed-module') -describe('ChannelManager', function() { - beforeEach(function() { - this.rclient = {}; - this.other_rclient = {}; - return this.ChannelManager = SandboxedModule.require(modulePath, { requires: { - "settings-sharelatex": (this.settings = {}), - "metrics-sharelatex": (this.metrics = {inc: sinon.stub(), summary: sinon.stub()}), - "logger-sharelatex": (this.logger = { log: sinon.stub(), warn: sinon.stub(), error: sinon.stub() }) - } - });}); +describe('ChannelManager', function () { + beforeEach(function () { + this.rclient = {} + this.other_rclient = {} + return (this.ChannelManager = SandboxedModule.require(modulePath, { + requires: { + 'settings-sharelatex': (this.settings = {}), + 'metrics-sharelatex': (this.metrics = { + inc: sinon.stub(), + summary: sinon.stub() + }), + 'logger-sharelatex': (this.logger = { + log: sinon.stub(), + warn: sinon.stub(), + error: sinon.stub() + }) + } + })) + }) - describe("subscribe", function() { + describe('subscribe', function () { + describe('when there is no existing subscription for this redis client', function () { + beforeEach(function (done) { + this.rclient.subscribe = sinon.stub().resolves() + this.ChannelManager.subscribe( + this.rclient, + 'applied-ops', + '1234567890abcdef' + ) + return setTimeout(done) + }) - describe("when there is no existing subscription for this redis client", function() { - beforeEach(function(done) { - this.rclient.subscribe = sinon.stub().resolves(); - this.ChannelManager.subscribe(this.rclient, "applied-ops", "1234567890abcdef"); - return setTimeout(done); - }); + return it('should subscribe to the redis channel', function () { + return this.rclient.subscribe + .calledWithExactly('applied-ops:1234567890abcdef') + .should.equal(true) + }) + }) - return it("should subscribe to the redis channel", function() { - return this.rclient.subscribe.calledWithExactly("applied-ops:1234567890abcdef").should.equal(true); - }); - }); + describe('when there is an existing subscription for this redis client', function () { + beforeEach(function (done) { + this.rclient.subscribe = sinon.stub().resolves() + this.ChannelManager.subscribe( + this.rclient, + 'applied-ops', + '1234567890abcdef' + ) + this.ChannelManager.subscribe( + this.rclient, + 'applied-ops', + '1234567890abcdef' + ) + return setTimeout(done) + }) - describe("when there is an existing subscription for this redis client", function() { - beforeEach(function(done) { - this.rclient.subscribe = sinon.stub().resolves(); - this.ChannelManager.subscribe(this.rclient, "applied-ops", "1234567890abcdef"); - this.ChannelManager.subscribe(this.rclient, "applied-ops", "1234567890abcdef"); - return setTimeout(done); - }); + return it('should subscribe to the redis channel again', function () { + return this.rclient.subscribe.callCount.should.equal(2) + }) + }) - return it("should subscribe to the redis channel again", function() { - return this.rclient.subscribe.callCount.should.equal(2); - }); - }); + describe('when subscribe errors', function () { + beforeEach(function (done) { + this.rclient.subscribe = sinon + .stub() + .onFirstCall() + .rejects(new Error('some redis error')) + .onSecondCall() + .resolves() + const p = this.ChannelManager.subscribe( + this.rclient, + 'applied-ops', + '1234567890abcdef' + ) + p.then(() => done(new Error('should not subscribe but fail'))).catch( + (err) => { + err.message.should.equal('some redis error') + this.ChannelManager.getClientMapEntry(this.rclient) + .has('applied-ops:1234567890abcdef') + .should.equal(false) + this.ChannelManager.subscribe( + this.rclient, + 'applied-ops', + '1234567890abcdef' + ) + // subscribe is wrapped in Promise, delay other assertions + return setTimeout(done) + } + ) + return null + }) - describe("when subscribe errors", function() { - beforeEach(function(done) { - this.rclient.subscribe = sinon.stub() - .onFirstCall().rejects(new Error("some redis error")) - .onSecondCall().resolves(); - const p = this.ChannelManager.subscribe(this.rclient, "applied-ops", "1234567890abcdef"); - p.then(() => done(new Error('should not subscribe but fail'))).catch(err => { - err.message.should.equal("some redis error"); - this.ChannelManager.getClientMapEntry(this.rclient).has("applied-ops:1234567890abcdef").should.equal(false); - this.ChannelManager.subscribe(this.rclient, "applied-ops", "1234567890abcdef"); - // subscribe is wrapped in Promise, delay other assertions - return setTimeout(done); - }); - return null; - }); + it('should have recorded the error', function () { + return expect( + this.metrics.inc.calledWithExactly('subscribe.failed.applied-ops') + ).to.equal(true) + }) - it("should have recorded the error", function() { - return expect(this.metrics.inc.calledWithExactly("subscribe.failed.applied-ops")).to.equal(true); - }); + it('should subscribe again', function () { + return this.rclient.subscribe.callCount.should.equal(2) + }) - it("should subscribe again", function() { - return this.rclient.subscribe.callCount.should.equal(2); - }); + return it('should cleanup', function () { + return this.ChannelManager.getClientMapEntry(this.rclient) + .has('applied-ops:1234567890abcdef') + .should.equal(false) + }) + }) - return it("should cleanup", function() { - return this.ChannelManager.getClientMapEntry(this.rclient).has("applied-ops:1234567890abcdef").should.equal(false); - }); - }); + describe('when subscribe errors and the clientChannelMap entry was replaced', function () { + beforeEach(function (done) { + this.rclient.subscribe = sinon + .stub() + .onFirstCall() + .rejects(new Error('some redis error')) + .onSecondCall() + .resolves() + this.first = this.ChannelManager.subscribe( + this.rclient, + 'applied-ops', + '1234567890abcdef' + ) + // ignore error + this.first.catch(() => {}) + expect( + this.ChannelManager.getClientMapEntry(this.rclient).get( + 'applied-ops:1234567890abcdef' + ) + ).to.equal(this.first) - describe("when subscribe errors and the clientChannelMap entry was replaced", function() { - beforeEach(function(done) { - this.rclient.subscribe = sinon.stub() - .onFirstCall().rejects(new Error("some redis error")) - .onSecondCall().resolves(); - this.first = this.ChannelManager.subscribe(this.rclient, "applied-ops", "1234567890abcdef"); - // ignore error - this.first.catch((() => {})); - expect(this.ChannelManager.getClientMapEntry(this.rclient).get("applied-ops:1234567890abcdef")).to.equal(this.first); + this.rclient.unsubscribe = sinon.stub().resolves() + this.ChannelManager.unsubscribe( + this.rclient, + 'applied-ops', + '1234567890abcdef' + ) + this.second = this.ChannelManager.subscribe( + this.rclient, + 'applied-ops', + '1234567890abcdef' + ) + // should get replaced immediately + expect( + this.ChannelManager.getClientMapEntry(this.rclient).get( + 'applied-ops:1234567890abcdef' + ) + ).to.equal(this.second) - this.rclient.unsubscribe = sinon.stub().resolves(); - this.ChannelManager.unsubscribe(this.rclient, "applied-ops", "1234567890abcdef"); - this.second = this.ChannelManager.subscribe(this.rclient, "applied-ops", "1234567890abcdef"); - // should get replaced immediately - expect(this.ChannelManager.getClientMapEntry(this.rclient).get("applied-ops:1234567890abcdef")).to.equal(this.second); + // let the first subscribe error -> unsubscribe -> subscribe + return setTimeout(done) + }) - // let the first subscribe error -> unsubscribe -> subscribe - return setTimeout(done); - }); + return it('should cleanup the second subscribePromise', function () { + return expect( + this.ChannelManager.getClientMapEntry(this.rclient).has( + 'applied-ops:1234567890abcdef' + ) + ).to.equal(false) + }) + }) - return it("should cleanup the second subscribePromise", function() { - return expect(this.ChannelManager.getClientMapEntry(this.rclient).has("applied-ops:1234567890abcdef")).to.equal(false); - }); - }); + return describe('when there is an existing subscription for another redis client but not this one', function () { + beforeEach(function (done) { + this.other_rclient.subscribe = sinon.stub().resolves() + this.ChannelManager.subscribe( + this.other_rclient, + 'applied-ops', + '1234567890abcdef' + ) + this.rclient.subscribe = sinon.stub().resolves() // discard the original stub + this.ChannelManager.subscribe( + this.rclient, + 'applied-ops', + '1234567890abcdef' + ) + return setTimeout(done) + }) - return describe("when there is an existing subscription for another redis client but not this one", function() { - beforeEach(function(done) { - this.other_rclient.subscribe = sinon.stub().resolves(); - this.ChannelManager.subscribe(this.other_rclient, "applied-ops", "1234567890abcdef"); - this.rclient.subscribe = sinon.stub().resolves(); // discard the original stub - this.ChannelManager.subscribe(this.rclient, "applied-ops", "1234567890abcdef"); - return setTimeout(done); - }); + return it('should subscribe to the redis channel on this redis client', function () { + return this.rclient.subscribe + .calledWithExactly('applied-ops:1234567890abcdef') + .should.equal(true) + }) + }) + }) - return it("should subscribe to the redis channel on this redis client", function() { - return this.rclient.subscribe.calledWithExactly("applied-ops:1234567890abcdef").should.equal(true); - }); - }); - }); + describe('unsubscribe', function () { + describe('when there is no existing subscription for this redis client', function () { + beforeEach(function (done) { + this.rclient.unsubscribe = sinon.stub().resolves() + this.ChannelManager.unsubscribe( + this.rclient, + 'applied-ops', + '1234567890abcdef' + ) + return setTimeout(done) + }) - describe("unsubscribe", function() { + return it('should unsubscribe from the redis channel', function () { + return this.rclient.unsubscribe.called.should.equal(true) + }) + }) - describe("when there is no existing subscription for this redis client", function() { - beforeEach(function(done) { - this.rclient.unsubscribe = sinon.stub().resolves(); - this.ChannelManager.unsubscribe(this.rclient, "applied-ops", "1234567890abcdef"); - return setTimeout(done); - }); + describe('when there is an existing subscription for this another redis client but not this one', function () { + beforeEach(function (done) { + this.other_rclient.subscribe = sinon.stub().resolves() + this.rclient.unsubscribe = sinon.stub().resolves() + this.ChannelManager.subscribe( + this.other_rclient, + 'applied-ops', + '1234567890abcdef' + ) + this.ChannelManager.unsubscribe( + this.rclient, + 'applied-ops', + '1234567890abcdef' + ) + return setTimeout(done) + }) - return it("should unsubscribe from the redis channel", function() { - return this.rclient.unsubscribe.called.should.equal(true); - }); - }); + return it('should still unsubscribe from the redis channel on this client', function () { + return this.rclient.unsubscribe.called.should.equal(true) + }) + }) + describe('when unsubscribe errors and completes', function () { + beforeEach(function (done) { + this.rclient.subscribe = sinon.stub().resolves() + this.ChannelManager.subscribe( + this.rclient, + 'applied-ops', + '1234567890abcdef' + ) + this.rclient.unsubscribe = sinon + .stub() + .rejects(new Error('some redis error')) + this.ChannelManager.unsubscribe( + this.rclient, + 'applied-ops', + '1234567890abcdef' + ) + setTimeout(done) + return null + }) - describe("when there is an existing subscription for this another redis client but not this one", function() { - beforeEach(function(done) { - this.other_rclient.subscribe = sinon.stub().resolves(); - this.rclient.unsubscribe = sinon.stub().resolves(); - this.ChannelManager.subscribe(this.other_rclient, "applied-ops", "1234567890abcdef"); - this.ChannelManager.unsubscribe(this.rclient, "applied-ops", "1234567890abcdef"); - return setTimeout(done); - }); + it('should have cleaned up', function () { + return this.ChannelManager.getClientMapEntry(this.rclient) + .has('applied-ops:1234567890abcdef') + .should.equal(false) + }) - return it("should still unsubscribe from the redis channel on this client", function() { - return this.rclient.unsubscribe.called.should.equal(true); - }); - }); + return it('should not error out when subscribing again', function (done) { + const p = this.ChannelManager.subscribe( + this.rclient, + 'applied-ops', + '1234567890abcdef' + ) + p.then(() => done()).catch(done) + return null + }) + }) - describe("when unsubscribe errors and completes", function() { - beforeEach(function(done) { - this.rclient.subscribe = sinon.stub().resolves(); - this.ChannelManager.subscribe(this.rclient, "applied-ops", "1234567890abcdef"); - this.rclient.unsubscribe = sinon.stub().rejects(new Error("some redis error")); - this.ChannelManager.unsubscribe(this.rclient, "applied-ops", "1234567890abcdef"); - setTimeout(done); - return null; - }); + describe('when unsubscribe errors and another client subscribes at the same time', function () { + beforeEach(function (done) { + this.rclient.subscribe = sinon.stub().resolves() + this.ChannelManager.subscribe( + this.rclient, + 'applied-ops', + '1234567890abcdef' + ) + let rejectSubscribe + this.rclient.unsubscribe = () => + new Promise((resolve, reject) => (rejectSubscribe = reject)) + this.ChannelManager.unsubscribe( + this.rclient, + 'applied-ops', + '1234567890abcdef' + ) - it("should have cleaned up", function() { - return this.ChannelManager.getClientMapEntry(this.rclient).has("applied-ops:1234567890abcdef").should.equal(false); - }); + setTimeout(() => { + // delay, actualUnsubscribe should not see the new subscribe request + this.ChannelManager.subscribe( + this.rclient, + 'applied-ops', + '1234567890abcdef' + ) + .then(() => setTimeout(done)) + .catch(done) + return setTimeout(() => + // delay, rejectSubscribe is not defined immediately + rejectSubscribe(new Error('redis error')) + ) + }) + return null + }) - return it("should not error out when subscribing again", function(done) { - const p = this.ChannelManager.subscribe(this.rclient, "applied-ops", "1234567890abcdef"); - p.then(() => done()).catch(done); - return null; - }); - }); + it('should have recorded the error', function () { + return expect( + this.metrics.inc.calledWithExactly('unsubscribe.failed.applied-ops') + ).to.equal(true) + }) - describe("when unsubscribe errors and another client subscribes at the same time", function() { - beforeEach(function(done) { - this.rclient.subscribe = sinon.stub().resolves(); - this.ChannelManager.subscribe(this.rclient, "applied-ops", "1234567890abcdef"); - let rejectSubscribe; - this.rclient.unsubscribe = () => new Promise((resolve, reject) => rejectSubscribe = reject); - this.ChannelManager.unsubscribe(this.rclient, "applied-ops", "1234567890abcdef"); + it('should have subscribed', function () { + return this.rclient.subscribe.called.should.equal(true) + }) - setTimeout(() => { - // delay, actualUnsubscribe should not see the new subscribe request - this.ChannelManager.subscribe(this.rclient, "applied-ops", "1234567890abcdef") - .then(() => setTimeout(done)).catch(done); - return setTimeout(() => // delay, rejectSubscribe is not defined immediately - rejectSubscribe(new Error("redis error"))); - }); - return null; - }); + return it('should have discarded the finished Promise', function () { + return this.ChannelManager.getClientMapEntry(this.rclient) + .has('applied-ops:1234567890abcdef') + .should.equal(false) + }) + }) - it("should have recorded the error", function() { - return expect(this.metrics.inc.calledWithExactly("unsubscribe.failed.applied-ops")).to.equal(true); - }); + return describe('when there is an existing subscription for this redis client', function () { + beforeEach(function (done) { + this.rclient.subscribe = sinon.stub().resolves() + this.rclient.unsubscribe = sinon.stub().resolves() + this.ChannelManager.subscribe( + this.rclient, + 'applied-ops', + '1234567890abcdef' + ) + this.ChannelManager.unsubscribe( + this.rclient, + 'applied-ops', + '1234567890abcdef' + ) + return setTimeout(done) + }) - it("should have subscribed", function() { - return this.rclient.subscribe.called.should.equal(true); - }); + return it('should unsubscribe from the redis channel', function () { + return this.rclient.unsubscribe + .calledWithExactly('applied-ops:1234567890abcdef') + .should.equal(true) + }) + }) + }) - return it("should have discarded the finished Promise", function() { - return this.ChannelManager.getClientMapEntry(this.rclient).has("applied-ops:1234567890abcdef").should.equal(false); - }); - }); + return describe('publish', function () { + describe("when the channel is 'all'", function () { + beforeEach(function () { + this.rclient.publish = sinon.stub() + return this.ChannelManager.publish( + this.rclient, + 'applied-ops', + 'all', + 'random-message' + ) + }) - return describe("when there is an existing subscription for this redis client", function() { - beforeEach(function(done) { - this.rclient.subscribe = sinon.stub().resolves(); - this.rclient.unsubscribe = sinon.stub().resolves(); - this.ChannelManager.subscribe(this.rclient, "applied-ops", "1234567890abcdef"); - this.ChannelManager.unsubscribe(this.rclient, "applied-ops", "1234567890abcdef"); - return setTimeout(done); - }); + return it('should publish on the base channel', function () { + return this.rclient.publish + .calledWithExactly('applied-ops', 'random-message') + .should.equal(true) + }) + }) - return it("should unsubscribe from the redis channel", function() { - return this.rclient.unsubscribe.calledWithExactly("applied-ops:1234567890abcdef").should.equal(true); - }); - }); - }); + describe('when the channel has an specific id', function () { + describe('when the individual channel setting is false', function () { + beforeEach(function () { + this.rclient.publish = sinon.stub() + this.settings.publishOnIndividualChannels = false + return this.ChannelManager.publish( + this.rclient, + 'applied-ops', + '1234567890abcdef', + 'random-message' + ) + }) - return describe("publish", function() { + return it('should publish on the per-id channel', function () { + this.rclient.publish + .calledWithExactly('applied-ops', 'random-message') + .should.equal(true) + return this.rclient.publish.calledOnce.should.equal(true) + }) + }) - describe("when the channel is 'all'", function() { - beforeEach(function() { - this.rclient.publish = sinon.stub(); - return this.ChannelManager.publish(this.rclient, "applied-ops", "all", "random-message"); - }); + return describe('when the individual channel setting is true', function () { + beforeEach(function () { + this.rclient.publish = sinon.stub() + this.settings.publishOnIndividualChannels = true + return this.ChannelManager.publish( + this.rclient, + 'applied-ops', + '1234567890abcdef', + 'random-message' + ) + }) - return it("should publish on the base channel", function() { - return this.rclient.publish.calledWithExactly("applied-ops", "random-message").should.equal(true); - }); - }); + return it('should publish on the per-id channel', function () { + this.rclient.publish + .calledWithExactly('applied-ops:1234567890abcdef', 'random-message') + .should.equal(true) + return this.rclient.publish.calledOnce.should.equal(true) + }) + }) + }) - describe("when the channel has an specific id", function() { + return describe('metrics', function () { + beforeEach(function () { + this.rclient.publish = sinon.stub() + return this.ChannelManager.publish( + this.rclient, + 'applied-ops', + 'all', + 'random-message' + ) + }) - describe("when the individual channel setting is false", function() { - beforeEach(function() { - this.rclient.publish = sinon.stub(); - this.settings.publishOnIndividualChannels = false; - return this.ChannelManager.publish(this.rclient, "applied-ops", "1234567890abcdef", "random-message"); - }); - - return it("should publish on the per-id channel", function() { - this.rclient.publish.calledWithExactly("applied-ops", "random-message").should.equal(true); - return this.rclient.publish.calledOnce.should.equal(true); - }); - }); - - return describe("when the individual channel setting is true", function() { - beforeEach(function() { - this.rclient.publish = sinon.stub(); - this.settings.publishOnIndividualChannels = true; - return this.ChannelManager.publish(this.rclient, "applied-ops", "1234567890abcdef", "random-message"); - }); - - return it("should publish on the per-id channel", function() { - this.rclient.publish.calledWithExactly("applied-ops:1234567890abcdef", "random-message").should.equal(true); - return this.rclient.publish.calledOnce.should.equal(true); - }); - }); - }); - - return describe("metrics", function() { - beforeEach(function() { - this.rclient.publish = sinon.stub(); - return this.ChannelManager.publish(this.rclient, "applied-ops", "all", "random-message"); - }); - - return it("should track the payload size", function() { - return this.metrics.summary.calledWithExactly( - "redis.publish.applied-ops", - "random-message".length - ).should.equal(true); - }); - }); - }); -}); + return it('should track the payload size', function () { + return this.metrics.summary + .calledWithExactly( + 'redis.publish.applied-ops', + 'random-message'.length + ) + .should.equal(true) + }) + }) + }) +}) diff --git a/services/real-time/test/unit/js/ConnectedUsersManagerTests.js b/services/real-time/test/unit/js/ConnectedUsersManagerTests.js index f6b026fd1a..8e84c41130 100644 --- a/services/real-time/test/unit/js/ConnectedUsersManagerTests.js +++ b/services/real-time/test/unit/js/ConnectedUsersManagerTests.js @@ -12,218 +12,398 @@ * Full docs: https://github.com/decaffeinate/decaffeinate/blob/master/docs/suggestions.md */ -const should = require('chai').should(); -const SandboxedModule = require('sandboxed-module'); -const assert = require('assert'); -const path = require('path'); -const sinon = require('sinon'); -const modulePath = path.join(__dirname, "../../../app/js/ConnectedUsersManager"); -const { - expect -} = require("chai"); -const tk = require("timekeeper"); +const should = require('chai').should() +const SandboxedModule = require('sandboxed-module') +const assert = require('assert') +const path = require('path') +const sinon = require('sinon') +const modulePath = path.join(__dirname, '../../../app/js/ConnectedUsersManager') +const { expect } = require('chai') +const tk = require('timekeeper') +describe('ConnectedUsersManager', function () { + beforeEach(function () { + this.settings = { + redis: { + realtime: { + key_schema: { + clientsInProject({ project_id }) { + return `clients_in_project:${project_id}` + }, + connectedUser({ project_id, client_id }) { + return `connected_user:${project_id}:${client_id}` + } + } + } + } + } + this.rClient = { + auth() {}, + setex: sinon.stub(), + sadd: sinon.stub(), + get: sinon.stub(), + srem: sinon.stub(), + del: sinon.stub(), + smembers: sinon.stub(), + expire: sinon.stub(), + hset: sinon.stub(), + hgetall: sinon.stub(), + exec: sinon.stub(), + multi: () => { + return this.rClient + } + } + tk.freeze(new Date()) -describe("ConnectedUsersManager", function() { + this.ConnectedUsersManager = SandboxedModule.require(modulePath, { + requires: { + 'settings-sharelatex': this.settings, + 'logger-sharelatex': { log() {} }, + 'redis-sharelatex': { + createClient: () => { + return this.rClient + } + } + } + }) + this.client_id = '32132132' + this.project_id = 'dskjh2u21321' + this.user = { + _id: 'user-id-123', + first_name: 'Joe', + last_name: 'Bloggs', + email: 'joe@example.com' + } + return (this.cursorData = { + row: 12, + column: 9, + doc_id: '53c3b8c85fee64000023dc6e' + }) + }) - beforeEach(function() { + afterEach(function () { + return tk.reset() + }) - this.settings = { - redis: { - realtime: { - key_schema: { - clientsInProject({project_id}) { return `clients_in_project:${project_id}`; }, - connectedUser({project_id, client_id}){ return `connected_user:${project_id}:${client_id}`; } - } - } - } - }; - this.rClient = { - auth() {}, - setex:sinon.stub(), - sadd:sinon.stub(), - get: sinon.stub(), - srem:sinon.stub(), - del:sinon.stub(), - smembers:sinon.stub(), - expire:sinon.stub(), - hset:sinon.stub(), - hgetall:sinon.stub(), - exec:sinon.stub(), - multi: () => { return this.rClient; } - }; - tk.freeze(new Date()); + describe('updateUserPosition', function () { + beforeEach(function () { + return this.rClient.exec.callsArgWith(0) + }) - this.ConnectedUsersManager = SandboxedModule.require(modulePath, { requires: { - "settings-sharelatex":this.settings, - "logger-sharelatex": { log() {} - }, - "redis-sharelatex": { createClient:() => { - return this.rClient; - } - } - } - } - ); - this.client_id = "32132132"; - this.project_id = "dskjh2u21321"; - this.user = { - _id: "user-id-123", - first_name: "Joe", - last_name: "Bloggs", - email: "joe@example.com" - }; - return this.cursorData = { row: 12, column: 9, doc_id: '53c3b8c85fee64000023dc6e' };}); + it('should set a key with the date and give it a ttl', function (done) { + return this.ConnectedUsersManager.updateUserPosition( + this.project_id, + this.client_id, + this.user, + null, + (err) => { + this.rClient.hset + .calledWith( + `connected_user:${this.project_id}:${this.client_id}`, + 'last_updated_at', + Date.now() + ) + .should.equal(true) + return done() + } + ) + }) - afterEach(function() { return tk.reset(); }); + it('should set a key with the user_id', function (done) { + return this.ConnectedUsersManager.updateUserPosition( + this.project_id, + this.client_id, + this.user, + null, + (err) => { + this.rClient.hset + .calledWith( + `connected_user:${this.project_id}:${this.client_id}`, + 'user_id', + this.user._id + ) + .should.equal(true) + return done() + } + ) + }) - describe("updateUserPosition", function() { - beforeEach(function() { - return this.rClient.exec.callsArgWith(0); - }); + it('should set a key with the first_name', function (done) { + return this.ConnectedUsersManager.updateUserPosition( + this.project_id, + this.client_id, + this.user, + null, + (err) => { + this.rClient.hset + .calledWith( + `connected_user:${this.project_id}:${this.client_id}`, + 'first_name', + this.user.first_name + ) + .should.equal(true) + return done() + } + ) + }) - it("should set a key with the date and give it a ttl", function(done){ - return this.ConnectedUsersManager.updateUserPosition(this.project_id, this.client_id, this.user, null, err=> { - this.rClient.hset.calledWith(`connected_user:${this.project_id}:${this.client_id}`, "last_updated_at", Date.now()).should.equal(true); - return done(); - }); - }); + it('should set a key with the last_name', function (done) { + return this.ConnectedUsersManager.updateUserPosition( + this.project_id, + this.client_id, + this.user, + null, + (err) => { + this.rClient.hset + .calledWith( + `connected_user:${this.project_id}:${this.client_id}`, + 'last_name', + this.user.last_name + ) + .should.equal(true) + return done() + } + ) + }) - it("should set a key with the user_id", function(done){ - return this.ConnectedUsersManager.updateUserPosition(this.project_id, this.client_id, this.user, null, err=> { - this.rClient.hset.calledWith(`connected_user:${this.project_id}:${this.client_id}`, "user_id", this.user._id).should.equal(true); - return done(); - }); - }); + it('should set a key with the email', function (done) { + return this.ConnectedUsersManager.updateUserPosition( + this.project_id, + this.client_id, + this.user, + null, + (err) => { + this.rClient.hset + .calledWith( + `connected_user:${this.project_id}:${this.client_id}`, + 'email', + this.user.email + ) + .should.equal(true) + return done() + } + ) + }) - it("should set a key with the first_name", function(done){ - return this.ConnectedUsersManager.updateUserPosition(this.project_id, this.client_id, this.user, null, err=> { - this.rClient.hset.calledWith(`connected_user:${this.project_id}:${this.client_id}`, "first_name", this.user.first_name).should.equal(true); - return done(); - }); - }); + it('should push the client_id on to the project list', function (done) { + return this.ConnectedUsersManager.updateUserPosition( + this.project_id, + this.client_id, + this.user, + null, + (err) => { + this.rClient.sadd + .calledWith(`clients_in_project:${this.project_id}`, this.client_id) + .should.equal(true) + return done() + } + ) + }) - it("should set a key with the last_name", function(done){ - return this.ConnectedUsersManager.updateUserPosition(this.project_id, this.client_id, this.user, null, err=> { - this.rClient.hset.calledWith(`connected_user:${this.project_id}:${this.client_id}`, "last_name", this.user.last_name).should.equal(true); - return done(); - }); - }); + it('should add a ttl to the project set so it stays clean', function (done) { + return this.ConnectedUsersManager.updateUserPosition( + this.project_id, + this.client_id, + this.user, + null, + (err) => { + this.rClient.expire + .calledWith( + `clients_in_project:${this.project_id}`, + 24 * 4 * 60 * 60 + ) + .should.equal(true) + return done() + } + ) + }) - it("should set a key with the email", function(done){ - return this.ConnectedUsersManager.updateUserPosition(this.project_id, this.client_id, this.user, null, err=> { - this.rClient.hset.calledWith(`connected_user:${this.project_id}:${this.client_id}`, "email", this.user.email).should.equal(true); - return done(); - }); - }); + it('should add a ttl to the connected user so it stays clean', function (done) { + return this.ConnectedUsersManager.updateUserPosition( + this.project_id, + this.client_id, + this.user, + null, + (err) => { + this.rClient.expire + .calledWith( + `connected_user:${this.project_id}:${this.client_id}`, + 60 * 15 + ) + .should.equal(true) + return done() + } + ) + }) - it("should push the client_id on to the project list", function(done){ - return this.ConnectedUsersManager.updateUserPosition(this.project_id, this.client_id, this.user, null, err=> { - this.rClient.sadd.calledWith(`clients_in_project:${this.project_id}`, this.client_id).should.equal(true); - return done(); - }); - }); + return it('should set the cursor position when provided', function (done) { + return this.ConnectedUsersManager.updateUserPosition( + this.project_id, + this.client_id, + this.user, + this.cursorData, + (err) => { + this.rClient.hset + .calledWith( + `connected_user:${this.project_id}:${this.client_id}`, + 'cursorData', + JSON.stringify(this.cursorData) + ) + .should.equal(true) + return done() + } + ) + }) + }) - it("should add a ttl to the project set so it stays clean", function(done){ - return this.ConnectedUsersManager.updateUserPosition(this.project_id, this.client_id, this.user, null, err=> { - this.rClient.expire.calledWith(`clients_in_project:${this.project_id}`, 24 * 4 * 60 * 60).should.equal(true); - return done(); - }); - }); + describe('markUserAsDisconnected', function () { + beforeEach(function () { + return this.rClient.exec.callsArgWith(0) + }) - it("should add a ttl to the connected user so it stays clean", function(done) { - return this.ConnectedUsersManager.updateUserPosition(this.project_id, this.client_id, this.user, null, err=> { - this.rClient.expire.calledWith(`connected_user:${this.project_id}:${this.client_id}`, 60 * 15).should.equal(true); - return done(); - }); - }); + it('should remove the user from the set', function (done) { + return this.ConnectedUsersManager.markUserAsDisconnected( + this.project_id, + this.client_id, + (err) => { + this.rClient.srem + .calledWith(`clients_in_project:${this.project_id}`, this.client_id) + .should.equal(true) + return done() + } + ) + }) - return it("should set the cursor position when provided", function(done){ - return this.ConnectedUsersManager.updateUserPosition(this.project_id, this.client_id, this.user, this.cursorData, err=> { - this.rClient.hset.calledWith(`connected_user:${this.project_id}:${this.client_id}`, "cursorData", JSON.stringify(this.cursorData)).should.equal(true); - return done(); - }); - }); - }); + it('should delete the connected_user string', function (done) { + return this.ConnectedUsersManager.markUserAsDisconnected( + this.project_id, + this.client_id, + (err) => { + this.rClient.del + .calledWith(`connected_user:${this.project_id}:${this.client_id}`) + .should.equal(true) + return done() + } + ) + }) - describe("markUserAsDisconnected", function() { - beforeEach(function() { - return this.rClient.exec.callsArgWith(0); - }); + return it('should add a ttl to the connected user set so it stays clean', function (done) { + return this.ConnectedUsersManager.markUserAsDisconnected( + this.project_id, + this.client_id, + (err) => { + this.rClient.expire + .calledWith( + `clients_in_project:${this.project_id}`, + 24 * 4 * 60 * 60 + ) + .should.equal(true) + return done() + } + ) + }) + }) - it("should remove the user from the set", function(done){ - return this.ConnectedUsersManager.markUserAsDisconnected(this.project_id, this.client_id, err=> { - this.rClient.srem.calledWith(`clients_in_project:${this.project_id}`, this.client_id).should.equal(true); - return done(); - }); - }); + describe('_getConnectedUser', function () { + it('should return a connected user if there is a user object', function (done) { + const cursorData = JSON.stringify({ cursorData: { row: 1 } }) + this.rClient.hgetall.callsArgWith(1, null, { + connected_at: new Date(), + user_id: this.user._id, + last_updated_at: `${Date.now()}`, + cursorData + }) + return this.ConnectedUsersManager._getConnectedUser( + this.project_id, + this.client_id, + (err, result) => { + result.connected.should.equal(true) + result.client_id.should.equal(this.client_id) + return done() + } + ) + }) - it("should delete the connected_user string", function(done){ - return this.ConnectedUsersManager.markUserAsDisconnected(this.project_id, this.client_id, err=> { - this.rClient.del.calledWith(`connected_user:${this.project_id}:${this.client_id}`).should.equal(true); - return done(); - }); - }); + it('should return a not connected user if there is no object', function (done) { + this.rClient.hgetall.callsArgWith(1, null, null) + return this.ConnectedUsersManager._getConnectedUser( + this.project_id, + this.client_id, + (err, result) => { + result.connected.should.equal(false) + result.client_id.should.equal(this.client_id) + return done() + } + ) + }) - return it("should add a ttl to the connected user set so it stays clean", function(done){ - return this.ConnectedUsersManager.markUserAsDisconnected(this.project_id, this.client_id, err=> { - this.rClient.expire.calledWith(`clients_in_project:${this.project_id}`, 24 * 4 * 60 * 60).should.equal(true); - return done(); - }); - }); - }); + return it('should return a not connected user if there is an empty object', function (done) { + this.rClient.hgetall.callsArgWith(1, null, {}) + return this.ConnectedUsersManager._getConnectedUser( + this.project_id, + this.client_id, + (err, result) => { + result.connected.should.equal(false) + result.client_id.should.equal(this.client_id) + return done() + } + ) + }) + }) - describe("_getConnectedUser", function() { - - it("should return a connected user if there is a user object", function(done){ - const cursorData = JSON.stringify({cursorData:{row:1}}); - this.rClient.hgetall.callsArgWith(1, null, {connected_at:new Date(), user_id: this.user._id, last_updated_at: `${Date.now()}`, cursorData}); - return this.ConnectedUsersManager._getConnectedUser(this.project_id, this.client_id, (err, result)=> { - result.connected.should.equal(true); - result.client_id.should.equal(this.client_id); - return done(); - }); - }); - - it("should return a not connected user if there is no object", function(done){ - this.rClient.hgetall.callsArgWith(1, null, null); - return this.ConnectedUsersManager._getConnectedUser(this.project_id, this.client_id, (err, result)=> { - result.connected.should.equal(false); - result.client_id.should.equal(this.client_id); - return done(); - }); - }); - - return it("should return a not connected user if there is an empty object", function(done){ - this.rClient.hgetall.callsArgWith(1, null, {}); - return this.ConnectedUsersManager._getConnectedUser(this.project_id, this.client_id, (err, result)=> { - result.connected.should.equal(false); - result.client_id.should.equal(this.client_id); - return done(); - }); - }); - }); - - return describe("getConnectedUsers", function() { - - beforeEach(function() { - this.users = ["1234", "5678", "9123", "8234"]; - this.rClient.smembers.callsArgWith(1, null, this.users); - this.ConnectedUsersManager._getConnectedUser = sinon.stub(); - this.ConnectedUsersManager._getConnectedUser.withArgs(this.project_id, this.users[0]).callsArgWith(2, null, {connected:true, client_age: 2, client_id:this.users[0]}); - this.ConnectedUsersManager._getConnectedUser.withArgs(this.project_id, this.users[1]).callsArgWith(2, null, {connected:false, client_age: 1, client_id:this.users[1]}); - this.ConnectedUsersManager._getConnectedUser.withArgs(this.project_id, this.users[2]).callsArgWith(2, null, {connected:true, client_age: 3, client_id:this.users[2]}); - return this.ConnectedUsersManager._getConnectedUser.withArgs(this.project_id, this.users[3]).callsArgWith(2, null, {connected:true, client_age: 11, client_id:this.users[3]}); - }); // connected but old - - return it("should only return the users in the list which are still in redis and recently updated", function(done){ - return this.ConnectedUsersManager.getConnectedUsers(this.project_id, (err, users)=> { - users.length.should.equal(2); - users[0].should.deep.equal({client_id:this.users[0], client_age: 2, connected:true}); - users[1].should.deep.equal({client_id:this.users[2], client_age: 3, connected:true}); - return done(); - }); - }); - }); -}); + return describe('getConnectedUsers', function () { + beforeEach(function () { + this.users = ['1234', '5678', '9123', '8234'] + this.rClient.smembers.callsArgWith(1, null, this.users) + this.ConnectedUsersManager._getConnectedUser = sinon.stub() + this.ConnectedUsersManager._getConnectedUser + .withArgs(this.project_id, this.users[0]) + .callsArgWith(2, null, { + connected: true, + client_age: 2, + client_id: this.users[0] + }) + this.ConnectedUsersManager._getConnectedUser + .withArgs(this.project_id, this.users[1]) + .callsArgWith(2, null, { + connected: false, + client_age: 1, + client_id: this.users[1] + }) + this.ConnectedUsersManager._getConnectedUser + .withArgs(this.project_id, this.users[2]) + .callsArgWith(2, null, { + connected: true, + client_age: 3, + client_id: this.users[2] + }) + return this.ConnectedUsersManager._getConnectedUser + .withArgs(this.project_id, this.users[3]) + .callsArgWith(2, null, { + connected: true, + client_age: 11, + client_id: this.users[3] + }) + }) // connected but old + return it('should only return the users in the list which are still in redis and recently updated', function (done) { + return this.ConnectedUsersManager.getConnectedUsers( + this.project_id, + (err, users) => { + users.length.should.equal(2) + users[0].should.deep.equal({ + client_id: this.users[0], + client_age: 2, + connected: true + }) + users[1].should.deep.equal({ + client_id: this.users[2], + client_age: 3, + connected: true + }) + return done() + } + ) + }) + }) +}) diff --git a/services/real-time/test/unit/js/DocumentUpdaterControllerTests.js b/services/real-time/test/unit/js/DocumentUpdaterControllerTests.js index 8b62b381a0..532346f359 100644 --- a/services/real-time/test/unit/js/DocumentUpdaterControllerTests.js +++ b/services/real-time/test/unit/js/DocumentUpdaterControllerTests.js @@ -10,200 +10,250 @@ * DS102: Remove unnecessary code created because of implicit returns * Full docs: https://github.com/decaffeinate/decaffeinate/blob/master/docs/suggestions.md */ -const SandboxedModule = require('sandboxed-module'); -const sinon = require('sinon'); -require('chai').should(); -const modulePath = require('path').join(__dirname, '../../../app/js/DocumentUpdaterController'); -const MockClient = require("./helpers/MockClient"); +const SandboxedModule = require('sandboxed-module') +const sinon = require('sinon') +require('chai').should() +const modulePath = require('path').join( + __dirname, + '../../../app/js/DocumentUpdaterController' +) +const MockClient = require('./helpers/MockClient') -describe("DocumentUpdaterController", function() { - beforeEach(function() { - this.project_id = "project-id-123"; - this.doc_id = "doc-id-123"; - this.callback = sinon.stub(); - this.io = { "mock": "socket.io" }; - this.rclient = []; - this.RoomEvents = { on: sinon.stub() }; - return this.EditorUpdatesController = SandboxedModule.require(modulePath, { requires: { - "logger-sharelatex": (this.logger = { error: sinon.stub(), log: sinon.stub(), warn: sinon.stub() }), - "settings-sharelatex": (this.settings = { - redis: { - documentupdater: { - key_schema: { - pendingUpdates({doc_id}) { return `PendingUpdates:${doc_id}`; } - } - }, - pubsub: null - } - }), - "redis-sharelatex" : (this.redis = { - createClient: name => { - let rclientStub; - this.rclient.push(rclientStub = {name}); - return rclientStub; - } - }), - "./SafeJsonParse": (this.SafeJsonParse = - {parse: (data, cb) => cb(null, JSON.parse(data))}), - "./EventLogger": (this.EventLogger = {checkEventOrder: sinon.stub()}), - "./HealthCheckManager": {check: sinon.stub()}, - "metrics-sharelatex": (this.metrics = {inc: sinon.stub()}), - "./RoomManager" : (this.RoomManager = { eventSource: sinon.stub().returns(this.RoomEvents)}), - "./ChannelManager": (this.ChannelManager = {}) - } - });}); +describe('DocumentUpdaterController', function () { + beforeEach(function () { + this.project_id = 'project-id-123' + this.doc_id = 'doc-id-123' + this.callback = sinon.stub() + this.io = { mock: 'socket.io' } + this.rclient = [] + this.RoomEvents = { on: sinon.stub() } + return (this.EditorUpdatesController = SandboxedModule.require(modulePath, { + requires: { + 'logger-sharelatex': (this.logger = { + error: sinon.stub(), + log: sinon.stub(), + warn: sinon.stub() + }), + 'settings-sharelatex': (this.settings = { + redis: { + documentupdater: { + key_schema: { + pendingUpdates({ doc_id }) { + return `PendingUpdates:${doc_id}` + } + } + }, + pubsub: null + } + }), + 'redis-sharelatex': (this.redis = { + createClient: (name) => { + let rclientStub + this.rclient.push((rclientStub = { name })) + return rclientStub + } + }), + './SafeJsonParse': (this.SafeJsonParse = { + parse: (data, cb) => cb(null, JSON.parse(data)) + }), + './EventLogger': (this.EventLogger = { checkEventOrder: sinon.stub() }), + './HealthCheckManager': { check: sinon.stub() }, + 'metrics-sharelatex': (this.metrics = { inc: sinon.stub() }), + './RoomManager': (this.RoomManager = { + eventSource: sinon.stub().returns(this.RoomEvents) + }), + './ChannelManager': (this.ChannelManager = {}) + } + })) + }) - describe("listenForUpdatesFromDocumentUpdater", function() { - beforeEach(function() { - this.rclient.length = 0; // clear any existing clients - this.EditorUpdatesController.rclientList = [this.redis.createClient("first"), this.redis.createClient("second")]; - this.rclient[0].subscribe = sinon.stub(); - this.rclient[0].on = sinon.stub(); - this.rclient[1].subscribe = sinon.stub(); - this.rclient[1].on = sinon.stub(); - return this.EditorUpdatesController.listenForUpdatesFromDocumentUpdater(); - }); - - it("should subscribe to the doc-updater stream", function() { - return this.rclient[0].subscribe.calledWith("applied-ops").should.equal(true); - }); + describe('listenForUpdatesFromDocumentUpdater', function () { + beforeEach(function () { + this.rclient.length = 0 // clear any existing clients + this.EditorUpdatesController.rclientList = [ + this.redis.createClient('first'), + this.redis.createClient('second') + ] + this.rclient[0].subscribe = sinon.stub() + this.rclient[0].on = sinon.stub() + this.rclient[1].subscribe = sinon.stub() + this.rclient[1].on = sinon.stub() + return this.EditorUpdatesController.listenForUpdatesFromDocumentUpdater() + }) - it("should register a callback to handle updates", function() { - return this.rclient[0].on.calledWith("message").should.equal(true); - }); + it('should subscribe to the doc-updater stream', function () { + return this.rclient[0].subscribe + .calledWith('applied-ops') + .should.equal(true) + }) - return it("should subscribe to any additional doc-updater stream", function() { - this.rclient[1].subscribe.calledWith("applied-ops").should.equal(true); - return this.rclient[1].on.calledWith("message").should.equal(true); - }); - }); + it('should register a callback to handle updates', function () { + return this.rclient[0].on.calledWith('message').should.equal(true) + }) - describe("_processMessageFromDocumentUpdater", function() { - describe("with bad JSON", function() { - beforeEach(function() { - this.SafeJsonParse.parse = sinon.stub().callsArgWith(1, new Error("oops")); - return this.EditorUpdatesController._processMessageFromDocumentUpdater(this.io, "applied-ops", "blah"); - }); - - return it("should log an error", function() { - return this.logger.error.called.should.equal(true); - }); - }); + return it('should subscribe to any additional doc-updater stream', function () { + this.rclient[1].subscribe.calledWith('applied-ops').should.equal(true) + return this.rclient[1].on.calledWith('message').should.equal(true) + }) + }) - describe("with update", function() { - beforeEach(function() { - this.message = { - doc_id: this.doc_id, - op: {t: "foo", p: 12} - }; - this.EditorUpdatesController._applyUpdateFromDocumentUpdater = sinon.stub(); - return this.EditorUpdatesController._processMessageFromDocumentUpdater(this.io, "applied-ops", JSON.stringify(this.message)); - }); + describe('_processMessageFromDocumentUpdater', function () { + describe('with bad JSON', function () { + beforeEach(function () { + this.SafeJsonParse.parse = sinon + .stub() + .callsArgWith(1, new Error('oops')) + return this.EditorUpdatesController._processMessageFromDocumentUpdater( + this.io, + 'applied-ops', + 'blah' + ) + }) - return it("should apply the update", function() { - return this.EditorUpdatesController._applyUpdateFromDocumentUpdater - .calledWith(this.io, this.doc_id, this.message.op) - .should.equal(true); - }); - }); + return it('should log an error', function () { + return this.logger.error.called.should.equal(true) + }) + }) - return describe("with error", function() { - beforeEach(function() { - this.message = { - doc_id: this.doc_id, - error: "Something went wrong" - }; - this.EditorUpdatesController._processErrorFromDocumentUpdater = sinon.stub(); - return this.EditorUpdatesController._processMessageFromDocumentUpdater(this.io, "applied-ops", JSON.stringify(this.message)); - }); + describe('with update', function () { + beforeEach(function () { + this.message = { + doc_id: this.doc_id, + op: { t: 'foo', p: 12 } + } + this.EditorUpdatesController._applyUpdateFromDocumentUpdater = sinon.stub() + return this.EditorUpdatesController._processMessageFromDocumentUpdater( + this.io, + 'applied-ops', + JSON.stringify(this.message) + ) + }) - return it("should process the error", function() { - return this.EditorUpdatesController._processErrorFromDocumentUpdater - .calledWith(this.io, this.doc_id, this.message.error) - .should.equal(true); - }); - }); - }); + return it('should apply the update', function () { + return this.EditorUpdatesController._applyUpdateFromDocumentUpdater + .calledWith(this.io, this.doc_id, this.message.op) + .should.equal(true) + }) + }) - describe("_applyUpdateFromDocumentUpdater", function() { - beforeEach(function() { - this.sourceClient = new MockClient(); - this.otherClients = [new MockClient(), new MockClient()]; - this.update = { - op: [ {t: "foo", p: 12} ], - meta: { source: this.sourceClient.publicId - }, - v: (this.version = 42), - doc: this.doc_id - }; - return this.io.sockets = - {clients: sinon.stub().returns([this.sourceClient, ...Array.from(this.otherClients), this.sourceClient])}; - }); // include a duplicate client - - describe("normally", function() { - beforeEach(function() { - return this.EditorUpdatesController._applyUpdateFromDocumentUpdater(this.io, this.doc_id, this.update); - }); + return describe('with error', function () { + beforeEach(function () { + this.message = { + doc_id: this.doc_id, + error: 'Something went wrong' + } + this.EditorUpdatesController._processErrorFromDocumentUpdater = sinon.stub() + return this.EditorUpdatesController._processMessageFromDocumentUpdater( + this.io, + 'applied-ops', + JSON.stringify(this.message) + ) + }) - it("should send a version bump to the source client", function() { - this.sourceClient.emit - .calledWith("otUpdateApplied", {v: this.version, doc: this.doc_id}) - .should.equal(true); - return this.sourceClient.emit.calledOnce.should.equal(true); - }); + return it('should process the error', function () { + return this.EditorUpdatesController._processErrorFromDocumentUpdater + .calledWith(this.io, this.doc_id, this.message.error) + .should.equal(true) + }) + }) + }) - it("should get the clients connected to the document", function() { - return this.io.sockets.clients - .calledWith(this.doc_id) - .should.equal(true); - }); + describe('_applyUpdateFromDocumentUpdater', function () { + beforeEach(function () { + this.sourceClient = new MockClient() + this.otherClients = [new MockClient(), new MockClient()] + this.update = { + op: [{ t: 'foo', p: 12 }], + meta: { source: this.sourceClient.publicId }, + v: (this.version = 42), + doc: this.doc_id + } + return (this.io.sockets = { + clients: sinon + .stub() + .returns([ + this.sourceClient, + ...Array.from(this.otherClients), + this.sourceClient + ]) + }) + }) // include a duplicate client - return it("should send the full update to the other clients", function() { - return Array.from(this.otherClients).map((client) => - client.emit - .calledWith("otUpdateApplied", this.update) - .should.equal(true)); - }); - }); - - return describe("with a duplicate op", function() { - beforeEach(function() { - this.update.dup = true; - return this.EditorUpdatesController._applyUpdateFromDocumentUpdater(this.io, this.doc_id, this.update); - }); - - it("should send a version bump to the source client as usual", function() { - return this.sourceClient.emit - .calledWith("otUpdateApplied", {v: this.version, doc: this.doc_id}) - .should.equal(true); - }); + describe('normally', function () { + beforeEach(function () { + return this.EditorUpdatesController._applyUpdateFromDocumentUpdater( + this.io, + this.doc_id, + this.update + ) + }) - return it("should not send anything to the other clients (they've already had the op)", function() { - return Array.from(this.otherClients).map((client) => - client.emit - .calledWith("otUpdateApplied") - .should.equal(false)); - }); - }); - }); + it('should send a version bump to the source client', function () { + this.sourceClient.emit + .calledWith('otUpdateApplied', { v: this.version, doc: this.doc_id }) + .should.equal(true) + return this.sourceClient.emit.calledOnce.should.equal(true) + }) - return describe("_processErrorFromDocumentUpdater", function() { - beforeEach(function() { - this.clients = [new MockClient(), new MockClient()]; - this.io.sockets = - {clients: sinon.stub().returns(this.clients)}; - return this.EditorUpdatesController._processErrorFromDocumentUpdater(this.io, this.doc_id, "Something went wrong"); - }); + it('should get the clients connected to the document', function () { + return this.io.sockets.clients + .calledWith(this.doc_id) + .should.equal(true) + }) - it("should log a warning", function() { - return this.logger.warn.called.should.equal(true); - }); + return it('should send the full update to the other clients', function () { + return Array.from(this.otherClients).map((client) => + client.emit + .calledWith('otUpdateApplied', this.update) + .should.equal(true) + ) + }) + }) - return it("should disconnect all clients in that document", function() { - this.io.sockets.clients.calledWith(this.doc_id).should.equal(true); - return Array.from(this.clients).map((client) => - client.disconnect.called.should.equal(true)); - }); - }); -}); + return describe('with a duplicate op', function () { + beforeEach(function () { + this.update.dup = true + return this.EditorUpdatesController._applyUpdateFromDocumentUpdater( + this.io, + this.doc_id, + this.update + ) + }) + it('should send a version bump to the source client as usual', function () { + return this.sourceClient.emit + .calledWith('otUpdateApplied', { v: this.version, doc: this.doc_id }) + .should.equal(true) + }) + + return it("should not send anything to the other clients (they've already had the op)", function () { + return Array.from(this.otherClients).map((client) => + client.emit.calledWith('otUpdateApplied').should.equal(false) + ) + }) + }) + }) + + return describe('_processErrorFromDocumentUpdater', function () { + beforeEach(function () { + this.clients = [new MockClient(), new MockClient()] + this.io.sockets = { clients: sinon.stub().returns(this.clients) } + return this.EditorUpdatesController._processErrorFromDocumentUpdater( + this.io, + this.doc_id, + 'Something went wrong' + ) + }) + + it('should log a warning', function () { + return this.logger.warn.called.should.equal(true) + }) + + return it('should disconnect all clients in that document', function () { + this.io.sockets.clients.calledWith(this.doc_id).should.equal(true) + return Array.from(this.clients).map((client) => + client.disconnect.called.should.equal(true) + ) + }) + }) +}) diff --git a/services/real-time/test/unit/js/DocumentUpdaterManagerTests.js b/services/real-time/test/unit/js/DocumentUpdaterManagerTests.js index 49b08fa2b2..dc42b52140 100644 --- a/services/real-time/test/unit/js/DocumentUpdaterManagerTests.js +++ b/services/real-time/test/unit/js/DocumentUpdaterManagerTests.js @@ -10,258 +10,372 @@ * DS102: Remove unnecessary code created because of implicit returns * Full docs: https://github.com/decaffeinate/decaffeinate/blob/master/docs/suggestions.md */ -require('chai').should(); -const sinon = require("sinon"); -const SandboxedModule = require('sandboxed-module'); -const path = require("path"); -const modulePath = '../../../app/js/DocumentUpdaterManager'; +require('chai').should() +const sinon = require('sinon') +const SandboxedModule = require('sandboxed-module') +const path = require('path') +const modulePath = '../../../app/js/DocumentUpdaterManager' -describe('DocumentUpdaterManager', function() { - beforeEach(function() { - let Timer; - this.project_id = "project-id-923"; - this.doc_id = "doc-id-394"; - this.lines = ["one", "two", "three"]; - this.version = 42; - this.settings = { - apis: { documentupdater: {url: "http://doc-updater.example.com"} - }, - redis: { documentupdater: { - key_schema: { - pendingUpdates({doc_id}) { return `PendingUpdates:${doc_id}`; } - } - } - }, - maxUpdateSize: 7 * 1024 * 1024 - }; - this.rclient = {auth() {}}; +describe('DocumentUpdaterManager', function () { + beforeEach(function () { + let Timer + this.project_id = 'project-id-923' + this.doc_id = 'doc-id-394' + this.lines = ['one', 'two', 'three'] + this.version = 42 + this.settings = { + apis: { documentupdater: { url: 'http://doc-updater.example.com' } }, + redis: { + documentupdater: { + key_schema: { + pendingUpdates({ doc_id }) { + return `PendingUpdates:${doc_id}` + } + } + } + }, + maxUpdateSize: 7 * 1024 * 1024 + } + this.rclient = { auth() {} } - return this.DocumentUpdaterManager = SandboxedModule.require(modulePath, { - requires: { - 'settings-sharelatex':this.settings, - 'logger-sharelatex': (this.logger = {log: sinon.stub(), error: sinon.stub(), warn: sinon.stub()}), - 'request': (this.request = {}), - 'redis-sharelatex' : { createClient: () => this.rclient - }, - 'metrics-sharelatex': (this.Metrics = { - summary: sinon.stub(), - Timer: (Timer = class Timer { - done() {} - }) - }) - }, - globals: { - JSON: (this.JSON = Object.create(JSON)) - } - } - ); - }); // avoid modifying JSON object directly + return (this.DocumentUpdaterManager = SandboxedModule.require(modulePath, { + requires: { + 'settings-sharelatex': this.settings, + 'logger-sharelatex': (this.logger = { + log: sinon.stub(), + error: sinon.stub(), + warn: sinon.stub() + }), + request: (this.request = {}), + 'redis-sharelatex': { createClient: () => this.rclient }, + 'metrics-sharelatex': (this.Metrics = { + summary: sinon.stub(), + Timer: (Timer = class Timer { + done() {} + }) + }) + }, + globals: { + JSON: (this.JSON = Object.create(JSON)) + } + })) + }) // avoid modifying JSON object directly - describe("getDocument", function() { - beforeEach(function() { - return this.callback = sinon.stub(); - }); + describe('getDocument', function () { + beforeEach(function () { + return (this.callback = sinon.stub()) + }) - describe("successfully", function() { - beforeEach(function() { - this.body = JSON.stringify({ - lines: this.lines, - version: this.version, - ops: (this.ops = ["mock-op-1", "mock-op-2"]), - ranges: (this.ranges = {"mock": "ranges"})}); - this.fromVersion = 2; - this.request.get = sinon.stub().callsArgWith(1, null, {statusCode: 200}, this.body); - return this.DocumentUpdaterManager.getDocument(this.project_id, this.doc_id, this.fromVersion, this.callback); - }); + describe('successfully', function () { + beforeEach(function () { + this.body = JSON.stringify({ + lines: this.lines, + version: this.version, + ops: (this.ops = ['mock-op-1', 'mock-op-2']), + ranges: (this.ranges = { mock: 'ranges' }) + }) + this.fromVersion = 2 + this.request.get = sinon + .stub() + .callsArgWith(1, null, { statusCode: 200 }, this.body) + return this.DocumentUpdaterManager.getDocument( + this.project_id, + this.doc_id, + this.fromVersion, + this.callback + ) + }) - it('should get the document from the document updater', function() { - const url = `${this.settings.apis.documentupdater.url}/project/${this.project_id}/doc/${this.doc_id}?fromVersion=${this.fromVersion}`; - return this.request.get.calledWith(url).should.equal(true); - }); + it('should get the document from the document updater', function () { + const url = `${this.settings.apis.documentupdater.url}/project/${this.project_id}/doc/${this.doc_id}?fromVersion=${this.fromVersion}` + return this.request.get.calledWith(url).should.equal(true) + }) - return it("should call the callback with the lines, version, ranges and ops", function() { - return this.callback.calledWith(null, this.lines, this.version, this.ranges, this.ops).should.equal(true); - }); - }); + return it('should call the callback with the lines, version, ranges and ops', function () { + return this.callback + .calledWith(null, this.lines, this.version, this.ranges, this.ops) + .should.equal(true) + }) + }) - describe("when the document updater API returns an error", function() { - beforeEach(function() { - this.request.get = sinon.stub().callsArgWith(1, (this.error = new Error("something went wrong")), null, null); - return this.DocumentUpdaterManager.getDocument(this.project_id, this.doc_id, this.fromVersion, this.callback); - }); + describe('when the document updater API returns an error', function () { + beforeEach(function () { + this.request.get = sinon + .stub() + .callsArgWith( + 1, + (this.error = new Error('something went wrong')), + null, + null + ) + return this.DocumentUpdaterManager.getDocument( + this.project_id, + this.doc_id, + this.fromVersion, + this.callback + ) + }) - return it("should return an error to the callback", function() { - return this.callback.calledWith(this.error).should.equal(true); - }); - }); + return it('should return an error to the callback', function () { + return this.callback.calledWith(this.error).should.equal(true) + }) + }) + ;[404, 422].forEach((statusCode) => + describe(`when the document updater returns a ${statusCode} status code`, function () { + beforeEach(function () { + this.request.get = sinon + .stub() + .callsArgWith(1, null, { statusCode }, '') + return this.DocumentUpdaterManager.getDocument( + this.project_id, + this.doc_id, + this.fromVersion, + this.callback + ) + }) - [404, 422].forEach(statusCode => describe(`when the document updater returns a ${statusCode} status code`, function() { - beforeEach(function() { - this.request.get = sinon.stub().callsArgWith(1, null, { statusCode }, ""); - return this.DocumentUpdaterManager.getDocument(this.project_id, this.doc_id, this.fromVersion, this.callback); - }); + return it('should return the callback with an error', function () { + this.callback.called.should.equal(true) + const err = this.callback.getCall(0).args[0] + err.should.have.property('statusCode', statusCode) + err.should.have.property( + 'message', + 'doc updater could not load requested ops' + ) + this.logger.error.called.should.equal(false) + return this.logger.warn.called.should.equal(true) + }) + }) + ) - return it("should return the callback with an error", function() { - this.callback.called.should.equal(true); - const err = this.callback.getCall(0).args[0]; - err.should.have.property('statusCode', statusCode); - err.should.have.property('message', "doc updater could not load requested ops"); - this.logger.error.called.should.equal(false); - return this.logger.warn.called.should.equal(true); - }); - })); + return describe('when the document updater returns a failure error code', function () { + beforeEach(function () { + this.request.get = sinon + .stub() + .callsArgWith(1, null, { statusCode: 500 }, '') + return this.DocumentUpdaterManager.getDocument( + this.project_id, + this.doc_id, + this.fromVersion, + this.callback + ) + }) - return describe("when the document updater returns a failure error code", function() { - beforeEach(function() { - this.request.get = sinon.stub().callsArgWith(1, null, { statusCode: 500 }, ""); - return this.DocumentUpdaterManager.getDocument(this.project_id, this.doc_id, this.fromVersion, this.callback); - }); + return it('should return the callback with an error', function () { + this.callback.called.should.equal(true) + const err = this.callback.getCall(0).args[0] + err.should.have.property('statusCode', 500) + err.should.have.property( + 'message', + 'doc updater returned a non-success status code: 500' + ) + return this.logger.error.called.should.equal(true) + }) + }) + }) - return it("should return the callback with an error", function() { - this.callback.called.should.equal(true); - const err = this.callback.getCall(0).args[0]; - err.should.have.property('statusCode', 500); - err.should.have.property('message', "doc updater returned a non-success status code: 500"); - return this.logger.error.called.should.equal(true); - }); - }); - }); + describe('flushProjectToMongoAndDelete', function () { + beforeEach(function () { + return (this.callback = sinon.stub()) + }) - describe('flushProjectToMongoAndDelete', function() { - beforeEach(function() { - return this.callback = sinon.stub(); - }); + describe('successfully', function () { + beforeEach(function () { + this.request.del = sinon + .stub() + .callsArgWith(1, null, { statusCode: 204 }, '') + return this.DocumentUpdaterManager.flushProjectToMongoAndDelete( + this.project_id, + this.callback + ) + }) - describe("successfully", function() { - beforeEach(function() { - this.request.del = sinon.stub().callsArgWith(1, null, {statusCode: 204}, ""); - return this.DocumentUpdaterManager.flushProjectToMongoAndDelete(this.project_id, this.callback); - }); + it('should delete the project from the document updater', function () { + const url = `${this.settings.apis.documentupdater.url}/project/${this.project_id}?background=true` + return this.request.del.calledWith(url).should.equal(true) + }) - it('should delete the project from the document updater', function() { - const url = `${this.settings.apis.documentupdater.url}/project/${this.project_id}?background=true`; - return this.request.del.calledWith(url).should.equal(true); - }); + return it('should call the callback with no error', function () { + return this.callback.calledWith(null).should.equal(true) + }) + }) - return it("should call the callback with no error", function() { - return this.callback.calledWith(null).should.equal(true); - }); - }); + describe('when the document updater API returns an error', function () { + beforeEach(function () { + this.request.del = sinon + .stub() + .callsArgWith( + 1, + (this.error = new Error('something went wrong')), + null, + null + ) + return this.DocumentUpdaterManager.flushProjectToMongoAndDelete( + this.project_id, + this.callback + ) + }) - describe("when the document updater API returns an error", function() { - beforeEach(function() { - this.request.del = sinon.stub().callsArgWith(1, (this.error = new Error("something went wrong")), null, null); - return this.DocumentUpdaterManager.flushProjectToMongoAndDelete(this.project_id, this.callback); - }); + return it('should return an error to the callback', function () { + return this.callback.calledWith(this.error).should.equal(true) + }) + }) - return it("should return an error to the callback", function() { - return this.callback.calledWith(this.error).should.equal(true); - }); - }); + return describe('when the document updater returns a failure error code', function () { + beforeEach(function () { + this.request.del = sinon + .stub() + .callsArgWith(1, null, { statusCode: 500 }, '') + return this.DocumentUpdaterManager.flushProjectToMongoAndDelete( + this.project_id, + this.callback + ) + }) - return describe("when the document updater returns a failure error code", function() { - beforeEach(function() { - this.request.del = sinon.stub().callsArgWith(1, null, { statusCode: 500 }, ""); - return this.DocumentUpdaterManager.flushProjectToMongoAndDelete(this.project_id, this.callback); - }); + return it('should return the callback with an error', function () { + this.callback.called.should.equal(true) + const err = this.callback.getCall(0).args[0] + err.should.have.property('statusCode', 500) + return err.should.have.property( + 'message', + 'document updater returned a failure status code: 500' + ) + }) + }) + }) - return it("should return the callback with an error", function() { - this.callback.called.should.equal(true); - const err = this.callback.getCall(0).args[0]; - err.should.have.property('statusCode', 500); - return err.should.have.property('message', "document updater returned a failure status code: 500"); - }); - }); - }); + return describe('queueChange', function () { + beforeEach(function () { + this.change = { + doc: '1234567890', + op: [{ d: 'test', p: 345 }], + v: 789 + } + this.rclient.rpush = sinon.stub().yields() + return (this.callback = sinon.stub()) + }) - return describe('queueChange', function() { - beforeEach(function() { - this.change = { - "doc":"1234567890", - "op":[{"d":"test", "p":345}], - "v": 789 - }; - this.rclient.rpush = sinon.stub().yields(); - return this.callback = sinon.stub(); - }); + describe('successfully', function () { + beforeEach(function () { + return this.DocumentUpdaterManager.queueChange( + this.project_id, + this.doc_id, + this.change, + this.callback + ) + }) - describe("successfully", function() { - beforeEach(function() { - return this.DocumentUpdaterManager.queueChange(this.project_id, this.doc_id, this.change, this.callback); - }); + it('should push the change', function () { + return this.rclient.rpush + .calledWith( + `PendingUpdates:${this.doc_id}`, + JSON.stringify(this.change) + ) + .should.equal(true) + }) - it("should push the change", function() { - return this.rclient.rpush - .calledWith(`PendingUpdates:${this.doc_id}`, JSON.stringify(this.change)) - .should.equal(true); - }); + return it('should notify the doc updater of the change via the pending-updates-list queue', function () { + return this.rclient.rpush + .calledWith( + 'pending-updates-list', + `${this.project_id}:${this.doc_id}` + ) + .should.equal(true) + }) + }) - return it("should notify the doc updater of the change via the pending-updates-list queue", function() { - return this.rclient.rpush - .calledWith("pending-updates-list", `${this.project_id}:${this.doc_id}`) - .should.equal(true); - }); - }); + describe('with error talking to redis during rpush', function () { + beforeEach(function () { + this.rclient.rpush = sinon + .stub() + .yields(new Error('something went wrong')) + return this.DocumentUpdaterManager.queueChange( + this.project_id, + this.doc_id, + this.change, + this.callback + ) + }) - describe("with error talking to redis during rpush", function() { - beforeEach(function() { - this.rclient.rpush = sinon.stub().yields(new Error("something went wrong")); - return this.DocumentUpdaterManager.queueChange(this.project_id, this.doc_id, this.change, this.callback); - }); + return it('should return an error', function () { + return this.callback + .calledWithExactly(sinon.match(Error)) + .should.equal(true) + }) + }) - return it("should return an error", function() { - return this.callback.calledWithExactly(sinon.match(Error)).should.equal(true); - }); - }); + describe('with null byte corruption', function () { + beforeEach(function () { + this.JSON.stringify = () => '["bad bytes! \u0000 <- here"]' + return this.DocumentUpdaterManager.queueChange( + this.project_id, + this.doc_id, + this.change, + this.callback + ) + }) - describe("with null byte corruption", function() { - beforeEach(function() { - this.JSON.stringify = () => '["bad bytes! \u0000 <- here"]'; - return this.DocumentUpdaterManager.queueChange(this.project_id, this.doc_id, this.change, this.callback); - }); + it('should return an error', function () { + return this.callback + .calledWithExactly(sinon.match(Error)) + .should.equal(true) + }) - it("should return an error", function() { - return this.callback.calledWithExactly(sinon.match(Error)).should.equal(true); - }); + return it('should not push the change onto the pending-updates-list queue', function () { + return this.rclient.rpush.called.should.equal(false) + }) + }) - return it("should not push the change onto the pending-updates-list queue", function() { - return this.rclient.rpush.called.should.equal(false); - }); - }); + describe('when the update is too large', function () { + beforeEach(function () { + this.change = { + op: { p: 12, t: 'update is too large'.repeat(1024 * 400) } + } + return this.DocumentUpdaterManager.queueChange( + this.project_id, + this.doc_id, + this.change, + this.callback + ) + }) - describe("when the update is too large", function() { - beforeEach(function() { - this.change = {op: {p: 12,t: "update is too large".repeat(1024 * 400)}}; - return this.DocumentUpdaterManager.queueChange(this.project_id, this.doc_id, this.change, this.callback); - }); + it('should return an error', function () { + return this.callback + .calledWithExactly(sinon.match(Error)) + .should.equal(true) + }) - it("should return an error", function() { - return this.callback.calledWithExactly(sinon.match(Error)).should.equal(true); - }); + it('should add the size to the error', function () { + return this.callback.args[0][0].updateSize.should.equal(7782422) + }) - it("should add the size to the error", function() { - return this.callback.args[0][0].updateSize.should.equal(7782422); - }); + return it('should not push the change onto the pending-updates-list queue', function () { + return this.rclient.rpush.called.should.equal(false) + }) + }) - return it("should not push the change onto the pending-updates-list queue", function() { - return this.rclient.rpush.called.should.equal(false); - }); - }); + return describe('with invalid keys', function () { + beforeEach(function () { + this.change = { + op: [{ d: 'test', p: 345 }], + version: 789 // not a valid key + } + return this.DocumentUpdaterManager.queueChange( + this.project_id, + this.doc_id, + this.change, + this.callback + ) + }) - return describe("with invalid keys", function() { - beforeEach(function() { - this.change = { - "op":[{"d":"test", "p":345}], - "version": 789 // not a valid key - }; - return this.DocumentUpdaterManager.queueChange(this.project_id, this.doc_id, this.change, this.callback); - }); - - return it("should remove the invalid keys from the change", function() { - return this.rclient.rpush - .calledWith(`PendingUpdates:${this.doc_id}`, JSON.stringify({op:this.change.op})) - .should.equal(true); - }); - }); - }); -}); + return it('should remove the invalid keys from the change', function () { + return this.rclient.rpush + .calledWith( + `PendingUpdates:${this.doc_id}`, + JSON.stringify({ op: this.change.op }) + ) + .should.equal(true) + }) + }) + }) +}) diff --git a/services/real-time/test/unit/js/DrainManagerTests.js b/services/real-time/test/unit/js/DrainManagerTests.js index 6d6c8b826e..7ed7f1e06e 100644 --- a/services/real-time/test/unit/js/DrainManagerTests.js +++ b/services/real-time/test/unit/js/DrainManagerTests.js @@ -9,111 +9,124 @@ * DS102: Remove unnecessary code created because of implicit returns * Full docs: https://github.com/decaffeinate/decaffeinate/blob/master/docs/suggestions.md */ -const should = require('chai').should(); -const sinon = require("sinon"); -const SandboxedModule = require('sandboxed-module'); -const path = require("path"); -const modulePath = path.join(__dirname, "../../../app/js/DrainManager"); +const should = require('chai').should() +const sinon = require('sinon') +const SandboxedModule = require('sandboxed-module') +const path = require('path') +const modulePath = path.join(__dirname, '../../../app/js/DrainManager') -describe("DrainManager", function() { - beforeEach(function() { - this.DrainManager = SandboxedModule.require(modulePath, { requires: { - "logger-sharelatex": (this.logger = {log: sinon.stub()}) - } - } - ); - return this.io = { - sockets: { - clients: sinon.stub() - } - }; - }); +describe('DrainManager', function () { + beforeEach(function () { + this.DrainManager = SandboxedModule.require(modulePath, { + requires: { + 'logger-sharelatex': (this.logger = { log: sinon.stub() }) + } + }) + return (this.io = { + sockets: { + clients: sinon.stub() + } + }) + }) - describe("startDrainTimeWindow", function() { - beforeEach(function() { - this.clients = []; - for (let i = 0; i <= 5399; i++) { - this.clients[i] = { - id: i, - emit: sinon.stub() - }; - } - this.io.sockets.clients.returns(this.clients); - return this.DrainManager.startDrain = sinon.stub(); - }); + describe('startDrainTimeWindow', function () { + beforeEach(function () { + this.clients = [] + for (let i = 0; i <= 5399; i++) { + this.clients[i] = { + id: i, + emit: sinon.stub() + } + } + this.io.sockets.clients.returns(this.clients) + return (this.DrainManager.startDrain = sinon.stub()) + }) - return it("should set a drain rate fast enough", function(done){ - this.DrainManager.startDrainTimeWindow(this.io, 9); - this.DrainManager.startDrain.calledWith(this.io, 10).should.equal(true); - return done(); - }); - }); + return it('should set a drain rate fast enough', function (done) { + this.DrainManager.startDrainTimeWindow(this.io, 9) + this.DrainManager.startDrain.calledWith(this.io, 10).should.equal(true) + return done() + }) + }) + return describe('reconnectNClients', function () { + beforeEach(function () { + this.clients = [] + for (let i = 0; i <= 9; i++) { + this.clients[i] = { + id: i, + emit: sinon.stub() + } + } + return this.io.sockets.clients.returns(this.clients) + }) - return describe("reconnectNClients", function() { - beforeEach(function() { - this.clients = []; - for (let i = 0; i <= 9; i++) { - this.clients[i] = { - id: i, - emit: sinon.stub() - }; - } - return this.io.sockets.clients.returns(this.clients); - }); + return describe('after first pass', function () { + beforeEach(function () { + return this.DrainManager.reconnectNClients(this.io, 3) + }) - return describe("after first pass", function() { - beforeEach(function() { - return this.DrainManager.reconnectNClients(this.io, 3); - }); - - it("should reconnect the first 3 clients", function() { - return [0, 1, 2].map((i) => - this.clients[i].emit.calledWith("reconnectGracefully").should.equal(true)); - }); - - it("should not reconnect any more clients", function() { - return [3, 4, 5, 6, 7, 8, 9].map((i) => - this.clients[i].emit.calledWith("reconnectGracefully").should.equal(false)); - }); - - return describe("after second pass", function() { - beforeEach(function() { - return this.DrainManager.reconnectNClients(this.io, 3); - }); - - it("should reconnect the next 3 clients", function() { - return [3, 4, 5].map((i) => - this.clients[i].emit.calledWith("reconnectGracefully").should.equal(true)); - }); - - it("should not reconnect any more clients", function() { - return [6, 7, 8, 9].map((i) => - this.clients[i].emit.calledWith("reconnectGracefully").should.equal(false)); - }); - - it("should not reconnect the first 3 clients again", function() { - return [0, 1, 2].map((i) => - this.clients[i].emit.calledOnce.should.equal(true)); - }); - - return describe("after final pass", function() { - beforeEach(function() { - return this.DrainManager.reconnectNClients(this.io, 100); - }); - - it("should not reconnect the first 6 clients again", function() { - return [0, 1, 2, 3, 4, 5].map((i) => - this.clients[i].emit.calledOnce.should.equal(true)); - }); - - return it("should log out that it reached the end", function() { - return this.logger.log - .calledWith("All clients have been told to reconnectGracefully") - .should.equal(true); - }); - }); - }); - }); - }); -}); + it('should reconnect the first 3 clients', function () { + return [0, 1, 2].map((i) => + this.clients[i].emit + .calledWith('reconnectGracefully') + .should.equal(true) + ) + }) + + it('should not reconnect any more clients', function () { + return [3, 4, 5, 6, 7, 8, 9].map((i) => + this.clients[i].emit + .calledWith('reconnectGracefully') + .should.equal(false) + ) + }) + + return describe('after second pass', function () { + beforeEach(function () { + return this.DrainManager.reconnectNClients(this.io, 3) + }) + + it('should reconnect the next 3 clients', function () { + return [3, 4, 5].map((i) => + this.clients[i].emit + .calledWith('reconnectGracefully') + .should.equal(true) + ) + }) + + it('should not reconnect any more clients', function () { + return [6, 7, 8, 9].map((i) => + this.clients[i].emit + .calledWith('reconnectGracefully') + .should.equal(false) + ) + }) + + it('should not reconnect the first 3 clients again', function () { + return [0, 1, 2].map((i) => + this.clients[i].emit.calledOnce.should.equal(true) + ) + }) + + return describe('after final pass', function () { + beforeEach(function () { + return this.DrainManager.reconnectNClients(this.io, 100) + }) + + it('should not reconnect the first 6 clients again', function () { + return [0, 1, 2, 3, 4, 5].map((i) => + this.clients[i].emit.calledOnce.should.equal(true) + ) + }) + + return it('should log out that it reached the end', function () { + return this.logger.log + .calledWith('All clients have been told to reconnectGracefully') + .should.equal(true) + }) + }) + }) + }) + }) +}) diff --git a/services/real-time/test/unit/js/EventLoggerTests.js b/services/real-time/test/unit/js/EventLoggerTests.js index 2d3b298e20..7152f92ce7 100644 --- a/services/real-time/test/unit/js/EventLoggerTests.js +++ b/services/real-time/test/unit/js/EventLoggerTests.js @@ -8,99 +8,150 @@ * DS102: Remove unnecessary code created because of implicit returns * Full docs: https://github.com/decaffeinate/decaffeinate/blob/master/docs/suggestions.md */ -require('chai').should(); -const { - expect -} = require("chai"); -const SandboxedModule = require('sandboxed-module'); -const modulePath = '../../../app/js/EventLogger'; -const sinon = require("sinon"); -const tk = require("timekeeper"); +require('chai').should() +const { expect } = require('chai') +const SandboxedModule = require('sandboxed-module') +const modulePath = '../../../app/js/EventLogger' +const sinon = require('sinon') +const tk = require('timekeeper') -describe('EventLogger', function() { - beforeEach(function() { - this.start = Date.now(); - tk.freeze(new Date(this.start)); - this.EventLogger = SandboxedModule.require(modulePath, { requires: { - "logger-sharelatex": (this.logger = {error: sinon.stub(), warn: sinon.stub()}), - "metrics-sharelatex": (this.metrics = {inc: sinon.stub()}) - } - }); - this.channel = "applied-ops"; - this.id_1 = "random-hostname:abc-1"; - this.message_1 = "message-1"; - this.id_2 = "random-hostname:abc-2"; - return this.message_2 = "message-2"; - }); +describe('EventLogger', function () { + beforeEach(function () { + this.start = Date.now() + tk.freeze(new Date(this.start)) + this.EventLogger = SandboxedModule.require(modulePath, { + requires: { + 'logger-sharelatex': (this.logger = { + error: sinon.stub(), + warn: sinon.stub() + }), + 'metrics-sharelatex': (this.metrics = { inc: sinon.stub() }) + } + }) + this.channel = 'applied-ops' + this.id_1 = 'random-hostname:abc-1' + this.message_1 = 'message-1' + this.id_2 = 'random-hostname:abc-2' + return (this.message_2 = 'message-2') + }) - afterEach(function() { return tk.reset(); }); + afterEach(function () { + return tk.reset() + }) - return describe('checkEventOrder', function() { + return describe('checkEventOrder', function () { + describe('when the events are in order', function () { + beforeEach(function () { + this.EventLogger.checkEventOrder( + this.channel, + this.id_1, + this.message_1 + ) + return (this.status = this.EventLogger.checkEventOrder( + this.channel, + this.id_2, + this.message_2 + )) + }) - describe('when the events are in order', function() { - beforeEach(function() { - this.EventLogger.checkEventOrder(this.channel, this.id_1, this.message_1); - return this.status = this.EventLogger.checkEventOrder(this.channel, this.id_2, this.message_2); - }); + it('should accept events in order', function () { + return expect(this.status).to.be.undefined + }) - it('should accept events in order', function() { - return expect(this.status).to.be.undefined; - }); + return it('should increment the valid event metric', function () { + return this.metrics.inc.calledWith(`event.${this.channel}.valid`, 1) + .should.equal.true + }) + }) - return it('should increment the valid event metric', function() { - return this.metrics.inc.calledWith(`event.${this.channel}.valid`, 1) - .should.equal.true; - }); - }); + describe('when there is a duplicate events', function () { + beforeEach(function () { + this.EventLogger.checkEventOrder( + this.channel, + this.id_1, + this.message_1 + ) + return (this.status = this.EventLogger.checkEventOrder( + this.channel, + this.id_1, + this.message_1 + )) + }) - describe('when there is a duplicate events', function() { - beforeEach(function() { - this.EventLogger.checkEventOrder(this.channel, this.id_1, this.message_1); - return this.status = this.EventLogger.checkEventOrder(this.channel, this.id_1, this.message_1); - }); + it('should return "duplicate" for the same event', function () { + return expect(this.status).to.equal('duplicate') + }) - it('should return "duplicate" for the same event', function() { - return expect(this.status).to.equal("duplicate"); - }); + return it('should increment the duplicate event metric', function () { + return this.metrics.inc.calledWith(`event.${this.channel}.duplicate`, 1) + .should.equal.true + }) + }) - return it('should increment the duplicate event metric', function() { - return this.metrics.inc.calledWith(`event.${this.channel}.duplicate`, 1) - .should.equal.true; - }); - }); + describe('when there are out of order events', function () { + beforeEach(function () { + this.EventLogger.checkEventOrder( + this.channel, + this.id_1, + this.message_1 + ) + this.EventLogger.checkEventOrder( + this.channel, + this.id_2, + this.message_2 + ) + return (this.status = this.EventLogger.checkEventOrder( + this.channel, + this.id_1, + this.message_1 + )) + }) - describe('when there are out of order events', function() { - beforeEach(function() { - this.EventLogger.checkEventOrder(this.channel, this.id_1, this.message_1); - this.EventLogger.checkEventOrder(this.channel, this.id_2, this.message_2); - return this.status = this.EventLogger.checkEventOrder(this.channel, this.id_1, this.message_1); - }); + it('should return "out-of-order" for the event', function () { + return expect(this.status).to.equal('out-of-order') + }) - it('should return "out-of-order" for the event', function() { - return expect(this.status).to.equal("out-of-order"); - }); + return it('should increment the out-of-order event metric', function () { + return this.metrics.inc.calledWith( + `event.${this.channel}.out-of-order`, + 1 + ).should.equal.true + }) + }) - return it('should increment the out-of-order event metric', function() { - return this.metrics.inc.calledWith(`event.${this.channel}.out-of-order`, 1) - .should.equal.true; - }); - }); - - return describe('after MAX_STALE_TIME_IN_MS', function() { return it('should flush old entries', function() { - let status; - this.EventLogger.MAX_EVENTS_BEFORE_CLEAN = 10; - this.EventLogger.checkEventOrder(this.channel, this.id_1, this.message_1); - for (let i = 1; i <= 8; i++) { - status = this.EventLogger.checkEventOrder(this.channel, this.id_1, this.message_1); - expect(status).to.equal("duplicate"); - } - // the next event should flush the old entries aboce - this.EventLogger.MAX_STALE_TIME_IN_MS=1000; - tk.freeze(new Date(this.start + (5 * 1000))); - // because we flushed the entries this should not be a duplicate - this.EventLogger.checkEventOrder(this.channel, 'other-1', this.message_2); - status = this.EventLogger.checkEventOrder(this.channel, this.id_1, this.message_1); - return expect(status).to.be.undefined; - }); }); - }); -}); \ No newline at end of file + return describe('after MAX_STALE_TIME_IN_MS', function () { + return it('should flush old entries', function () { + let status + this.EventLogger.MAX_EVENTS_BEFORE_CLEAN = 10 + this.EventLogger.checkEventOrder( + this.channel, + this.id_1, + this.message_1 + ) + for (let i = 1; i <= 8; i++) { + status = this.EventLogger.checkEventOrder( + this.channel, + this.id_1, + this.message_1 + ) + expect(status).to.equal('duplicate') + } + // the next event should flush the old entries aboce + this.EventLogger.MAX_STALE_TIME_IN_MS = 1000 + tk.freeze(new Date(this.start + 5 * 1000)) + // because we flushed the entries this should not be a duplicate + this.EventLogger.checkEventOrder( + this.channel, + 'other-1', + this.message_2 + ) + status = this.EventLogger.checkEventOrder( + this.channel, + this.id_1, + this.message_1 + ) + return expect(status).to.be.undefined + }) + }) + }) +}) diff --git a/services/real-time/test/unit/js/RoomManagerTests.js b/services/real-time/test/unit/js/RoomManagerTests.js index b356d0ee5e..3aee509af5 100644 --- a/services/real-time/test/unit/js/RoomManagerTests.js +++ b/services/real-time/test/unit/js/RoomManagerTests.js @@ -10,357 +10,410 @@ * DS102: Remove unnecessary code created because of implicit returns * Full docs: https://github.com/decaffeinate/decaffeinate/blob/master/docs/suggestions.md */ -const chai = require('chai'); -const { - expect -} = chai; -const should = chai.should(); -const sinon = require("sinon"); -const modulePath = "../../../app/js/RoomManager.js"; -const SandboxedModule = require('sandboxed-module'); +const chai = require('chai') +const { expect } = chai +const should = chai.should() +const sinon = require('sinon') +const modulePath = '../../../app/js/RoomManager.js' +const SandboxedModule = require('sandboxed-module') -describe('RoomManager', function() { - beforeEach(function() { - this.project_id = "project-id-123"; - this.doc_id = "doc-id-456"; - this.other_doc_id = "doc-id-789"; - this.client = {namespace: {name: ''}, id: "first-client"}; - this.RoomManager = SandboxedModule.require(modulePath, { requires: { - "settings-sharelatex": (this.settings = {}), - "logger-sharelatex": (this.logger = { log: sinon.stub(), warn: sinon.stub(), error: sinon.stub() }), - "metrics-sharelatex": (this.metrics = { gauge: sinon.stub() }) - } - }); - this.RoomManager._clientsInRoom = sinon.stub(); - this.RoomManager._clientAlreadyInRoom = sinon.stub(); - this.RoomEvents = this.RoomManager.eventSource(); - sinon.spy(this.RoomEvents, 'emit'); - return sinon.spy(this.RoomEvents, 'once'); - }); - - describe("emitOnCompletion", function() { return describe("when a subscribe errors", function() { - afterEach(function() { - return process.removeListener("unhandledRejection", this.onUnhandled); - }); +describe('RoomManager', function () { + beforeEach(function () { + this.project_id = 'project-id-123' + this.doc_id = 'doc-id-456' + this.other_doc_id = 'doc-id-789' + this.client = { namespace: { name: '' }, id: 'first-client' } + this.RoomManager = SandboxedModule.require(modulePath, { + requires: { + 'settings-sharelatex': (this.settings = {}), + 'logger-sharelatex': (this.logger = { + log: sinon.stub(), + warn: sinon.stub(), + error: sinon.stub() + }), + 'metrics-sharelatex': (this.metrics = { gauge: sinon.stub() }) + } + }) + this.RoomManager._clientsInRoom = sinon.stub() + this.RoomManager._clientAlreadyInRoom = sinon.stub() + this.RoomEvents = this.RoomManager.eventSource() + sinon.spy(this.RoomEvents, 'emit') + return sinon.spy(this.RoomEvents, 'once') + }) - beforeEach(function(done) { - this.onUnhandled = error => { - this.unhandledError = error; - return done(new Error(`unhandledRejection: ${error.message}`)); - }; - process.on("unhandledRejection", this.onUnhandled); + describe('emitOnCompletion', function () { + return describe('when a subscribe errors', function () { + afterEach(function () { + return process.removeListener('unhandledRejection', this.onUnhandled) + }) - let reject; - const subscribePromise = new Promise((_, r) => reject = r); - const promises = [subscribePromise]; - const eventName = "project-subscribed-123"; - this.RoomEvents.once(eventName, () => setTimeout(done, 100)); - this.RoomManager.emitOnCompletion(promises, eventName); - return setTimeout(() => reject(new Error("subscribe failed"))); - }); + beforeEach(function (done) { + this.onUnhandled = (error) => { + this.unhandledError = error + return done(new Error(`unhandledRejection: ${error.message}`)) + } + process.on('unhandledRejection', this.onUnhandled) - return it("should keep going", function() { - return expect(this.unhandledError).to.not.exist; - }); - }); }); + let reject + const subscribePromise = new Promise((_, r) => (reject = r)) + const promises = [subscribePromise] + const eventName = 'project-subscribed-123' + this.RoomEvents.once(eventName, () => setTimeout(done, 100)) + this.RoomManager.emitOnCompletion(promises, eventName) + return setTimeout(() => reject(new Error('subscribe failed'))) + }) - describe("joinProject", function() { - - describe("when the project room is empty", function() { + return it('should keep going', function () { + return expect(this.unhandledError).to.not.exist + }) + }) + }) - beforeEach(function(done) { - this.RoomManager._clientsInRoom - .withArgs(this.client, this.project_id) - .onFirstCall().returns(0); - this.client.join = sinon.stub(); - this.callback = sinon.stub(); - this.RoomEvents.on('project-active', id => { - return setTimeout(() => { - return this.RoomEvents.emit(`project-subscribed-${id}`); - } - , 100); - }); - return this.RoomManager.joinProject(this.client, this.project_id, err => { - this.callback(err); - return done(); - }); - }); + describe('joinProject', function () { + describe('when the project room is empty', function () { + beforeEach(function (done) { + this.RoomManager._clientsInRoom + .withArgs(this.client, this.project_id) + .onFirstCall() + .returns(0) + this.client.join = sinon.stub() + this.callback = sinon.stub() + this.RoomEvents.on('project-active', (id) => { + return setTimeout(() => { + return this.RoomEvents.emit(`project-subscribed-${id}`) + }, 100) + }) + return this.RoomManager.joinProject( + this.client, + this.project_id, + (err) => { + this.callback(err) + return done() + } + ) + }) - it("should emit a 'project-active' event with the id", function() { - return this.RoomEvents.emit.calledWithExactly('project-active', this.project_id).should.equal(true); - }); + it("should emit a 'project-active' event with the id", function () { + return this.RoomEvents.emit + .calledWithExactly('project-active', this.project_id) + .should.equal(true) + }) - it("should listen for the 'project-subscribed-id' event", function() { - return this.RoomEvents.once.calledWith(`project-subscribed-${this.project_id}`).should.equal(true); - }); + it("should listen for the 'project-subscribed-id' event", function () { + return this.RoomEvents.once + .calledWith(`project-subscribed-${this.project_id}`) + .should.equal(true) + }) - return it("should join the room using the id", function() { - return this.client.join.calledWithExactly(this.project_id).should.equal(true); - }); - }); + return it('should join the room using the id', function () { + return this.client.join + .calledWithExactly(this.project_id) + .should.equal(true) + }) + }) - return describe("when there are other clients in the project room", function() { + return describe('when there are other clients in the project room', function () { + beforeEach(function () { + this.RoomManager._clientsInRoom + .withArgs(this.client, this.project_id) + .onFirstCall() + .returns(123) + .onSecondCall() + .returns(124) + this.client.join = sinon.stub() + return this.RoomManager.joinProject(this.client, this.project_id) + }) - beforeEach(function() { - this.RoomManager._clientsInRoom - .withArgs(this.client, this.project_id) - .onFirstCall().returns(123) - .onSecondCall().returns(124); - this.client.join = sinon.stub(); - return this.RoomManager.joinProject(this.client, this.project_id); - }); + it('should join the room using the id', function () { + return this.client.join.called.should.equal(true) + }) - it("should join the room using the id", function() { - return this.client.join.called.should.equal(true); - }); + return it('should not emit any events', function () { + return this.RoomEvents.emit.called.should.equal(false) + }) + }) + }) - return it("should not emit any events", function() { - return this.RoomEvents.emit.called.should.equal(false); - }); - }); - }); + describe('joinDoc', function () { + describe('when the doc room is empty', function () { + beforeEach(function (done) { + this.RoomManager._clientsInRoom + .withArgs(this.client, this.doc_id) + .onFirstCall() + .returns(0) + this.client.join = sinon.stub() + this.callback = sinon.stub() + this.RoomEvents.on('doc-active', (id) => { + return setTimeout(() => { + return this.RoomEvents.emit(`doc-subscribed-${id}`) + }, 100) + }) + return this.RoomManager.joinDoc(this.client, this.doc_id, (err) => { + this.callback(err) + return done() + }) + }) + it("should emit a 'doc-active' event with the id", function () { + return this.RoomEvents.emit + .calledWithExactly('doc-active', this.doc_id) + .should.equal(true) + }) - describe("joinDoc", function() { + it("should listen for the 'doc-subscribed-id' event", function () { + return this.RoomEvents.once + .calledWith(`doc-subscribed-${this.doc_id}`) + .should.equal(true) + }) - describe("when the doc room is empty", function() { + return it('should join the room using the id', function () { + return this.client.join + .calledWithExactly(this.doc_id) + .should.equal(true) + }) + }) - beforeEach(function(done) { - this.RoomManager._clientsInRoom - .withArgs(this.client, this.doc_id) - .onFirstCall().returns(0); - this.client.join = sinon.stub(); - this.callback = sinon.stub(); - this.RoomEvents.on('doc-active', id => { - return setTimeout(() => { - return this.RoomEvents.emit(`doc-subscribed-${id}`); - } - , 100); - }); - return this.RoomManager.joinDoc(this.client, this.doc_id, err => { - this.callback(err); - return done(); - }); - }); + return describe('when there are other clients in the doc room', function () { + beforeEach(function () { + this.RoomManager._clientsInRoom + .withArgs(this.client, this.doc_id) + .onFirstCall() + .returns(123) + .onSecondCall() + .returns(124) + this.client.join = sinon.stub() + return this.RoomManager.joinDoc(this.client, this.doc_id) + }) - it("should emit a 'doc-active' event with the id", function() { - return this.RoomEvents.emit.calledWithExactly('doc-active', this.doc_id).should.equal(true); - }); + it('should join the room using the id', function () { + return this.client.join.called.should.equal(true) + }) - it("should listen for the 'doc-subscribed-id' event", function() { - return this.RoomEvents.once.calledWith(`doc-subscribed-${this.doc_id}`).should.equal(true); - }); + return it('should not emit any events', function () { + return this.RoomEvents.emit.called.should.equal(false) + }) + }) + }) - return it("should join the room using the id", function() { - return this.client.join.calledWithExactly(this.doc_id).should.equal(true); - }); - }); + describe('leaveDoc', function () { + describe('when doc room will be empty after this client has left', function () { + beforeEach(function () { + this.RoomManager._clientAlreadyInRoom + .withArgs(this.client, this.doc_id) + .returns(true) + this.RoomManager._clientsInRoom + .withArgs(this.client, this.doc_id) + .onCall(0) + .returns(0) + this.client.leave = sinon.stub() + return this.RoomManager.leaveDoc(this.client, this.doc_id) + }) - return describe("when there are other clients in the doc room", function() { + it('should leave the room using the id', function () { + return this.client.leave + .calledWithExactly(this.doc_id) + .should.equal(true) + }) - beforeEach(function() { - this.RoomManager._clientsInRoom - .withArgs(this.client, this.doc_id) - .onFirstCall().returns(123) - .onSecondCall().returns(124); - this.client.join = sinon.stub(); - return this.RoomManager.joinDoc(this.client, this.doc_id); - }); + return it("should emit a 'doc-empty' event with the id", function () { + return this.RoomEvents.emit + .calledWithExactly('doc-empty', this.doc_id) + .should.equal(true) + }) + }) - it("should join the room using the id", function() { - return this.client.join.called.should.equal(true); - }); + describe('when there are other clients in the doc room', function () { + beforeEach(function () { + this.RoomManager._clientAlreadyInRoom + .withArgs(this.client, this.doc_id) + .returns(true) + this.RoomManager._clientsInRoom + .withArgs(this.client, this.doc_id) + .onCall(0) + .returns(123) + this.client.leave = sinon.stub() + return this.RoomManager.leaveDoc(this.client, this.doc_id) + }) - return it("should not emit any events", function() { - return this.RoomEvents.emit.called.should.equal(false); - }); - }); - }); + it('should leave the room using the id', function () { + return this.client.leave + .calledWithExactly(this.doc_id) + .should.equal(true) + }) + return it('should not emit any events', function () { + return this.RoomEvents.emit.called.should.equal(false) + }) + }) - describe("leaveDoc", function() { + return describe('when the client is not in the doc room', function () { + beforeEach(function () { + this.RoomManager._clientAlreadyInRoom + .withArgs(this.client, this.doc_id) + .returns(false) + this.RoomManager._clientsInRoom + .withArgs(this.client, this.doc_id) + .onCall(0) + .returns(0) + this.client.leave = sinon.stub() + return this.RoomManager.leaveDoc(this.client, this.doc_id) + }) - describe("when doc room will be empty after this client has left", function() { + it('should not leave the room', function () { + return this.client.leave.called.should.equal(false) + }) - beforeEach(function() { - this.RoomManager._clientAlreadyInRoom - .withArgs(this.client, this.doc_id) - .returns(true); - this.RoomManager._clientsInRoom - .withArgs(this.client, this.doc_id) - .onCall(0).returns(0); - this.client.leave = sinon.stub(); - return this.RoomManager.leaveDoc(this.client, this.doc_id); - }); + return it('should not emit any events', function () { + return this.RoomEvents.emit.called.should.equal(false) + }) + }) + }) - it("should leave the room using the id", function() { - return this.client.leave.calledWithExactly(this.doc_id).should.equal(true); - }); + return describe('leaveProjectAndDocs', function () { + return describe('when the client is connected to the project and multiple docs', function () { + beforeEach(function () { + this.RoomManager._roomsClientIsIn = sinon + .stub() + .returns([this.project_id, this.doc_id, this.other_doc_id]) + this.client.join = sinon.stub() + return (this.client.leave = sinon.stub()) + }) - return it("should emit a 'doc-empty' event with the id", function() { - return this.RoomEvents.emit.calledWithExactly('doc-empty', this.doc_id).should.equal(true); - }); - }); + describe('when this is the only client connected', function () { + beforeEach(function (done) { + // first call is for the join, + // second for the leave + this.RoomManager._clientsInRoom + .withArgs(this.client, this.doc_id) + .onCall(0) + .returns(0) + .onCall(1) + .returns(0) + this.RoomManager._clientsInRoom + .withArgs(this.client, this.other_doc_id) + .onCall(0) + .returns(0) + .onCall(1) + .returns(0) + this.RoomManager._clientsInRoom + .withArgs(this.client, this.project_id) + .onCall(0) + .returns(0) + .onCall(1) + .returns(0) + this.RoomManager._clientAlreadyInRoom + .withArgs(this.client, this.doc_id) + .returns(true) + .withArgs(this.client, this.other_doc_id) + .returns(true) + .withArgs(this.client, this.project_id) + .returns(true) + this.RoomEvents.on('project-active', (id) => { + return setTimeout(() => { + return this.RoomEvents.emit(`project-subscribed-${id}`) + }, 100) + }) + this.RoomEvents.on('doc-active', (id) => { + return setTimeout(() => { + return this.RoomEvents.emit(`doc-subscribed-${id}`) + }, 100) + }) + // put the client in the rooms + return this.RoomManager.joinProject( + this.client, + this.project_id, + () => { + return this.RoomManager.joinDoc(this.client, this.doc_id, () => { + return this.RoomManager.joinDoc( + this.client, + this.other_doc_id, + () => { + // now leave the project + this.RoomManager.leaveProjectAndDocs(this.client) + return done() + } + ) + }) + } + ) + }) + it('should leave all the docs', function () { + this.client.leave.calledWithExactly(this.doc_id).should.equal(true) + return this.client.leave + .calledWithExactly(this.other_doc_id) + .should.equal(true) + }) - describe("when there are other clients in the doc room", function() { + it('should leave the project', function () { + return this.client.leave + .calledWithExactly(this.project_id) + .should.equal(true) + }) - beforeEach(function() { - this.RoomManager._clientAlreadyInRoom - .withArgs(this.client, this.doc_id) - .returns(true); - this.RoomManager._clientsInRoom - .withArgs(this.client, this.doc_id) - .onCall(0).returns(123); - this.client.leave = sinon.stub(); - return this.RoomManager.leaveDoc(this.client, this.doc_id); - }); + it("should emit a 'doc-empty' event with the id for each doc", function () { + this.RoomEvents.emit + .calledWithExactly('doc-empty', this.doc_id) + .should.equal(true) + return this.RoomEvents.emit + .calledWithExactly('doc-empty', this.other_doc_id) + .should.equal(true) + }) - it("should leave the room using the id", function() { - return this.client.leave.calledWithExactly(this.doc_id).should.equal(true); - }); + return it("should emit a 'project-empty' event with the id for the project", function () { + return this.RoomEvents.emit + .calledWithExactly('project-empty', this.project_id) + .should.equal(true) + }) + }) - return it("should not emit any events", function() { - return this.RoomEvents.emit.called.should.equal(false); - }); - }); + return describe('when other clients are still connected', function () { + beforeEach(function () { + this.RoomManager._clientsInRoom + .withArgs(this.client, this.doc_id) + .onFirstCall() + .returns(123) + .onSecondCall() + .returns(122) + this.RoomManager._clientsInRoom + .withArgs(this.client, this.other_doc_id) + .onFirstCall() + .returns(123) + .onSecondCall() + .returns(122) + this.RoomManager._clientsInRoom + .withArgs(this.client, this.project_id) + .onFirstCall() + .returns(123) + .onSecondCall() + .returns(122) + this.RoomManager._clientAlreadyInRoom + .withArgs(this.client, this.doc_id) + .returns(true) + .withArgs(this.client, this.other_doc_id) + .returns(true) + .withArgs(this.client, this.project_id) + .returns(true) + return this.RoomManager.leaveProjectAndDocs(this.client) + }) - return describe("when the client is not in the doc room", function() { + it('should leave all the docs', function () { + this.client.leave.calledWithExactly(this.doc_id).should.equal(true) + return this.client.leave + .calledWithExactly(this.other_doc_id) + .should.equal(true) + }) - beforeEach(function() { - this.RoomManager._clientAlreadyInRoom - .withArgs(this.client, this.doc_id) - .returns(false); - this.RoomManager._clientsInRoom - .withArgs(this.client, this.doc_id) - .onCall(0).returns(0); - this.client.leave = sinon.stub(); - return this.RoomManager.leaveDoc(this.client, this.doc_id); - }); + it('should leave the project', function () { + return this.client.leave + .calledWithExactly(this.project_id) + .should.equal(true) + }) - it("should not leave the room", function() { - return this.client.leave.called.should.equal(false); - }); - - return it("should not emit any events", function() { - return this.RoomEvents.emit.called.should.equal(false); - }); - }); - }); - - - return describe("leaveProjectAndDocs", function() { return describe("when the client is connected to the project and multiple docs", function() { - - beforeEach(function() { - this.RoomManager._roomsClientIsIn = sinon.stub().returns([this.project_id, this.doc_id, this.other_doc_id]); - this.client.join = sinon.stub(); - return this.client.leave = sinon.stub(); - }); - - describe("when this is the only client connected", function() { - - beforeEach(function(done) { - // first call is for the join, - // second for the leave - this.RoomManager._clientsInRoom - .withArgs(this.client, this.doc_id) - .onCall(0).returns(0) - .onCall(1).returns(0); - this.RoomManager._clientsInRoom - .withArgs(this.client, this.other_doc_id) - .onCall(0).returns(0) - .onCall(1).returns(0); - this.RoomManager._clientsInRoom - .withArgs(this.client, this.project_id) - .onCall(0).returns(0) - .onCall(1).returns(0); - this.RoomManager._clientAlreadyInRoom - .withArgs(this.client, this.doc_id) - .returns(true) - .withArgs(this.client, this.other_doc_id) - .returns(true) - .withArgs(this.client, this.project_id) - .returns(true); - this.RoomEvents.on('project-active', id => { - return setTimeout(() => { - return this.RoomEvents.emit(`project-subscribed-${id}`); - } - , 100); - }); - this.RoomEvents.on('doc-active', id => { - return setTimeout(() => { - return this.RoomEvents.emit(`doc-subscribed-${id}`); - } - , 100); - }); - // put the client in the rooms - return this.RoomManager.joinProject(this.client, this.project_id, () => { - return this.RoomManager.joinDoc(this.client, this.doc_id, () => { - return this.RoomManager.joinDoc(this.client, this.other_doc_id, () => { - // now leave the project - this.RoomManager.leaveProjectAndDocs(this.client); - return done(); - }); - }); - }); - }); - - it("should leave all the docs", function() { - this.client.leave.calledWithExactly(this.doc_id).should.equal(true); - return this.client.leave.calledWithExactly(this.other_doc_id).should.equal(true); - }); - - it("should leave the project", function() { - return this.client.leave.calledWithExactly(this.project_id).should.equal(true); - }); - - it("should emit a 'doc-empty' event with the id for each doc", function() { - this.RoomEvents.emit.calledWithExactly('doc-empty', this.doc_id).should.equal(true); - return this.RoomEvents.emit.calledWithExactly('doc-empty', this.other_doc_id).should.equal(true); - }); - - return it("should emit a 'project-empty' event with the id for the project", function() { - return this.RoomEvents.emit.calledWithExactly('project-empty', this.project_id).should.equal(true); - }); - }); - - return describe("when other clients are still connected", function() { - - beforeEach(function() { - this.RoomManager._clientsInRoom - .withArgs(this.client, this.doc_id) - .onFirstCall().returns(123) - .onSecondCall().returns(122); - this.RoomManager._clientsInRoom - .withArgs(this.client, this.other_doc_id) - .onFirstCall().returns(123) - .onSecondCall().returns(122); - this.RoomManager._clientsInRoom - .withArgs(this.client, this.project_id) - .onFirstCall().returns(123) - .onSecondCall().returns(122); - this.RoomManager._clientAlreadyInRoom - .withArgs(this.client, this.doc_id) - .returns(true) - .withArgs(this.client, this.other_doc_id) - .returns(true) - .withArgs(this.client, this.project_id) - .returns(true); - return this.RoomManager.leaveProjectAndDocs(this.client); - }); - - it("should leave all the docs", function() { - this.client.leave.calledWithExactly(this.doc_id).should.equal(true); - return this.client.leave.calledWithExactly(this.other_doc_id).should.equal(true); - }); - - it("should leave the project", function() { - return this.client.leave.calledWithExactly(this.project_id).should.equal(true); - }); - - return it("should not emit any events", function() { - return this.RoomEvents.emit.called.should.equal(false); - }); - }); - }); }); -}); \ No newline at end of file + return it('should not emit any events', function () { + return this.RoomEvents.emit.called.should.equal(false) + }) + }) + }) + }) +}) diff --git a/services/real-time/test/unit/js/SafeJsonParseTest.js b/services/real-time/test/unit/js/SafeJsonParseTest.js index 58fed31397..4fb558a6b0 100644 --- a/services/real-time/test/unit/js/SafeJsonParseTest.js +++ b/services/real-time/test/unit/js/SafeJsonParseTest.js @@ -11,49 +11,49 @@ * DS102: Remove unnecessary code created because of implicit returns * Full docs: https://github.com/decaffeinate/decaffeinate/blob/master/docs/suggestions.md */ -require('chai').should(); -const { - expect -} = require("chai"); -const SandboxedModule = require('sandboxed-module'); -const modulePath = '../../../app/js/SafeJsonParse'; -const sinon = require("sinon"); +require('chai').should() +const { expect } = require('chai') +const SandboxedModule = require('sandboxed-module') +const modulePath = '../../../app/js/SafeJsonParse' +const sinon = require('sinon') -describe('SafeJsonParse', function() { - beforeEach(function() { - return this.SafeJsonParse = SandboxedModule.require(modulePath, { requires: { - "settings-sharelatex": (this.Settings = { - maxUpdateSize: 16 * 1024 - }), - "logger-sharelatex": (this.logger = {error: sinon.stub()}) - } - });}); +describe('SafeJsonParse', function () { + beforeEach(function () { + return (this.SafeJsonParse = SandboxedModule.require(modulePath, { + requires: { + 'settings-sharelatex': (this.Settings = { + maxUpdateSize: 16 * 1024 + }), + 'logger-sharelatex': (this.logger = { error: sinon.stub() }) + } + })) + }) - return describe("parse", function() { - it("should parse documents correctly", function(done) { - return this.SafeJsonParse.parse('{"foo": "bar"}', (error, parsed) => { - expect(parsed).to.deep.equal({foo: "bar"}); - return done(); - }); - }); - - it("should return an error on bad data", function(done) { - return this.SafeJsonParse.parse('blah', (error, parsed) => { - expect(error).to.exist; - return done(); - }); - }); - - return it("should return an error on oversized data", function(done) { - // we have a 2k overhead on top of max size - const big_blob = Array(16*1024).join("A"); - const data = `{\"foo\": \"${big_blob}\"}`; - this.Settings.maxUpdateSize = 2 * 1024; - return this.SafeJsonParse.parse(data, (error, parsed) => { - this.logger.error.called.should.equal(true); - expect(error).to.exist; - return done(); - }); - }); - }); -}); \ No newline at end of file + return describe('parse', function () { + it('should parse documents correctly', function (done) { + return this.SafeJsonParse.parse('{"foo": "bar"}', (error, parsed) => { + expect(parsed).to.deep.equal({ foo: 'bar' }) + return done() + }) + }) + + it('should return an error on bad data', function (done) { + return this.SafeJsonParse.parse('blah', (error, parsed) => { + expect(error).to.exist + return done() + }) + }) + + return it('should return an error on oversized data', function (done) { + // we have a 2k overhead on top of max size + const big_blob = Array(16 * 1024).join('A') + const data = `{\"foo\": \"${big_blob}\"}` + this.Settings.maxUpdateSize = 2 * 1024 + return this.SafeJsonParse.parse(data, (error, parsed) => { + this.logger.error.called.should.equal(true) + expect(error).to.exist + return done() + }) + }) + }) +}) diff --git a/services/real-time/test/unit/js/SessionSocketsTests.js b/services/real-time/test/unit/js/SessionSocketsTests.js index a57a58bfac..f4ae34bf78 100644 --- a/services/real-time/test/unit/js/SessionSocketsTests.js +++ b/services/real-time/test/unit/js/SessionSocketsTests.js @@ -9,168 +9,189 @@ * DS102: Remove unnecessary code created because of implicit returns * Full docs: https://github.com/decaffeinate/decaffeinate/blob/master/docs/suggestions.md */ -const {EventEmitter} = require('events'); -const {expect} = require('chai'); -const SandboxedModule = require('sandboxed-module'); -const modulePath = '../../../app/js/SessionSockets'; -const sinon = require('sinon'); +const { EventEmitter } = require('events') +const { expect } = require('chai') +const SandboxedModule = require('sandboxed-module') +const modulePath = '../../../app/js/SessionSockets' +const sinon = require('sinon') -describe('SessionSockets', function() { - before(function() { - this.SessionSocketsModule = SandboxedModule.require(modulePath); - this.io = new EventEmitter(); - this.id1 = Math.random().toString(); - this.id2 = Math.random().toString(); - const redisResponses = { - error: [new Error('Redis: something went wrong'), null], - unknownId: [null, null] - }; - redisResponses[this.id1] = [null, {user: {_id: '123'}}]; - redisResponses[this.id2] = [null, {user: {_id: 'abc'}}]; +describe('SessionSockets', function () { + before(function () { + this.SessionSocketsModule = SandboxedModule.require(modulePath) + this.io = new EventEmitter() + this.id1 = Math.random().toString() + this.id2 = Math.random().toString() + const redisResponses = { + error: [new Error('Redis: something went wrong'), null], + unknownId: [null, null] + } + redisResponses[this.id1] = [null, { user: { _id: '123' } }] + redisResponses[this.id2] = [null, { user: { _id: 'abc' } }] - this.sessionStore = { - get: sinon.stub().callsFake((id, fn) => fn.apply(null, redisResponses[id])) - }; - this.cookieParser = function(req, res, next) { - req.signedCookies = req._signedCookies; - return next(); - }; - this.SessionSockets = this.SessionSocketsModule(this.io, this.sessionStore, this.cookieParser, 'ol.sid'); - return this.checkSocket = (socket, fn) => { - this.SessionSockets.once('connection', fn); - return this.io.emit('connection', socket); - }; - }); + this.sessionStore = { + get: sinon + .stub() + .callsFake((id, fn) => fn.apply(null, redisResponses[id])) + } + this.cookieParser = function (req, res, next) { + req.signedCookies = req._signedCookies + return next() + } + this.SessionSockets = this.SessionSocketsModule( + this.io, + this.sessionStore, + this.cookieParser, + 'ol.sid' + ) + return (this.checkSocket = (socket, fn) => { + this.SessionSockets.once('connection', fn) + return this.io.emit('connection', socket) + }) + }) - describe('without cookies', function() { - before(function() { - return this.socket = {handshake: {}};}); + describe('without cookies', function () { + before(function () { + return (this.socket = { handshake: {} }) + }) - it('should return a lookup error', function(done) { - return this.checkSocket(this.socket, (error) => { - expect(error).to.exist; - expect(error.message).to.equal('could not look up session by key'); - return done(); - }); - }); + it('should return a lookup error', function (done) { + return this.checkSocket(this.socket, (error) => { + expect(error).to.exist + expect(error.message).to.equal('could not look up session by key') + return done() + }) + }) - return it('should not query redis', function(done) { - return this.checkSocket(this.socket, () => { - expect(this.sessionStore.get.called).to.equal(false); - return done(); - }); - }); - }); + return it('should not query redis', function (done) { + return this.checkSocket(this.socket, () => { + expect(this.sessionStore.get.called).to.equal(false) + return done() + }) + }) + }) - describe('with a different cookie', function() { - before(function() { - return this.socket = {handshake: {_signedCookies: {other: 1}}};}); + describe('with a different cookie', function () { + before(function () { + return (this.socket = { handshake: { _signedCookies: { other: 1 } } }) + }) - it('should return a lookup error', function(done) { - return this.checkSocket(this.socket, (error) => { - expect(error).to.exist; - expect(error.message).to.equal('could not look up session by key'); - return done(); - }); - }); + it('should return a lookup error', function (done) { + return this.checkSocket(this.socket, (error) => { + expect(error).to.exist + expect(error.message).to.equal('could not look up session by key') + return done() + }) + }) - return it('should not query redis', function(done) { - return this.checkSocket(this.socket, () => { - expect(this.sessionStore.get.called).to.equal(false); - return done(); - }); - }); - }); + return it('should not query redis', function (done) { + return this.checkSocket(this.socket, () => { + expect(this.sessionStore.get.called).to.equal(false) + return done() + }) + }) + }) - describe('with a valid cookie and a failing session lookup', function() { - before(function() { - return this.socket = {handshake: {_signedCookies: {'ol.sid': 'error'}}};}); + describe('with a valid cookie and a failing session lookup', function () { + before(function () { + return (this.socket = { + handshake: { _signedCookies: { 'ol.sid': 'error' } } + }) + }) - it('should query redis', function(done) { - return this.checkSocket(this.socket, () => { - expect(this.sessionStore.get.called).to.equal(true); - return done(); - }); - }); + it('should query redis', function (done) { + return this.checkSocket(this.socket, () => { + expect(this.sessionStore.get.called).to.equal(true) + return done() + }) + }) - return it('should return a redis error', function(done) { - return this.checkSocket(this.socket, (error) => { - expect(error).to.exist; - expect(error.message).to.equal('Redis: something went wrong'); - return done(); - }); - }); - }); + return it('should return a redis error', function (done) { + return this.checkSocket(this.socket, (error) => { + expect(error).to.exist + expect(error.message).to.equal('Redis: something went wrong') + return done() + }) + }) + }) - describe('with a valid cookie and no matching session', function() { - before(function() { - return this.socket = {handshake: {_signedCookies: {'ol.sid': 'unknownId'}}};}); + describe('with a valid cookie and no matching session', function () { + before(function () { + return (this.socket = { + handshake: { _signedCookies: { 'ol.sid': 'unknownId' } } + }) + }) - it('should query redis', function(done) { - return this.checkSocket(this.socket, () => { - expect(this.sessionStore.get.called).to.equal(true); - return done(); - }); - }); + it('should query redis', function (done) { + return this.checkSocket(this.socket, () => { + expect(this.sessionStore.get.called).to.equal(true) + return done() + }) + }) - return it('should return a lookup error', function(done) { - return this.checkSocket(this.socket, (error) => { - expect(error).to.exist; - expect(error.message).to.equal('could not look up session by key'); - return done(); - }); - }); - }); + return it('should return a lookup error', function (done) { + return this.checkSocket(this.socket, (error) => { + expect(error).to.exist + expect(error.message).to.equal('could not look up session by key') + return done() + }) + }) + }) - describe('with a valid cookie and a matching session', function() { - before(function() { - return this.socket = {handshake: {_signedCookies: {'ol.sid': this.id1}}};}); + describe('with a valid cookie and a matching session', function () { + before(function () { + return (this.socket = { + handshake: { _signedCookies: { 'ol.sid': this.id1 } } + }) + }) - it('should query redis', function(done) { - return this.checkSocket(this.socket, () => { - expect(this.sessionStore.get.called).to.equal(true); - return done(); - }); - }); + it('should query redis', function (done) { + return this.checkSocket(this.socket, () => { + expect(this.sessionStore.get.called).to.equal(true) + return done() + }) + }) - it('should not return an error', function(done) { - return this.checkSocket(this.socket, (error) => { - expect(error).to.not.exist; - return done(); - }); - }); + it('should not return an error', function (done) { + return this.checkSocket(this.socket, (error) => { + expect(error).to.not.exist + return done() + }) + }) - return it('should return the session', function(done) { - return this.checkSocket(this.socket, (error, s, session) => { - expect(session).to.deep.equal({user: {_id: '123'}}); - return done(); - }); - }); - }); + return it('should return the session', function (done) { + return this.checkSocket(this.socket, (error, s, session) => { + expect(session).to.deep.equal({ user: { _id: '123' } }) + return done() + }) + }) + }) - return describe('with a different valid cookie and matching session', function() { - before(function() { - return this.socket = {handshake: {_signedCookies: {'ol.sid': this.id2}}};}); + return describe('with a different valid cookie and matching session', function () { + before(function () { + return (this.socket = { + handshake: { _signedCookies: { 'ol.sid': this.id2 } } + }) + }) - it('should query redis', function(done) { - return this.checkSocket(this.socket, () => { - expect(this.sessionStore.get.called).to.equal(true); - return done(); - }); - }); + it('should query redis', function (done) { + return this.checkSocket(this.socket, () => { + expect(this.sessionStore.get.called).to.equal(true) + return done() + }) + }) - it('should not return an error', function(done) { - return this.checkSocket(this.socket, (error) => { - expect(error).to.not.exist; - return done(); - }); - }); + it('should not return an error', function (done) { + return this.checkSocket(this.socket, (error) => { + expect(error).to.not.exist + return done() + }) + }) - return it('should return the other session', function(done) { - return this.checkSocket(this.socket, (error, s, session) => { - expect(session).to.deep.equal({user: {_id: 'abc'}}); - return done(); - }); - }); - }); -}); + return it('should return the other session', function (done) { + return this.checkSocket(this.socket, (error, s, session) => { + expect(session).to.deep.equal({ user: { _id: 'abc' } }) + return done() + }) + }) + }) +}) diff --git a/services/real-time/test/unit/js/WebApiManagerTests.js b/services/real-time/test/unit/js/WebApiManagerTests.js index c868cbaf0e..2d792b563b 100644 --- a/services/real-time/test/unit/js/WebApiManagerTests.js +++ b/services/real-time/test/unit/js/WebApiManagerTests.js @@ -9,109 +9,154 @@ * DS102: Remove unnecessary code created because of implicit returns * Full docs: https://github.com/decaffeinate/decaffeinate/blob/master/docs/suggestions.md */ -const chai = require('chai'); -const should = chai.should(); -const sinon = require("sinon"); -const modulePath = "../../../app/js/WebApiManager.js"; -const SandboxedModule = require('sandboxed-module'); -const { CodedError } = require('../../../app/js/Errors'); +const chai = require('chai') +const should = chai.should() +const sinon = require('sinon') +const modulePath = '../../../app/js/WebApiManager.js' +const SandboxedModule = require('sandboxed-module') +const { CodedError } = require('../../../app/js/Errors') -describe('WebApiManager', function() { - beforeEach(function() { - this.project_id = "project-id-123"; - this.user_id = "user-id-123"; - this.user = {_id: this.user_id}; - this.callback = sinon.stub(); - return this.WebApiManager = SandboxedModule.require(modulePath, { requires: { - "request": (this.request = {}), - "settings-sharelatex": (this.settings = { - apis: { - web: { - url: "http://web.example.com", - user: "username", - pass: "password" - } - } - }), - "logger-sharelatex": (this.logger = { log: sinon.stub(), error: sinon.stub() }) - } - });}); +describe('WebApiManager', function () { + beforeEach(function () { + this.project_id = 'project-id-123' + this.user_id = 'user-id-123' + this.user = { _id: this.user_id } + this.callback = sinon.stub() + return (this.WebApiManager = SandboxedModule.require(modulePath, { + requires: { + request: (this.request = {}), + 'settings-sharelatex': (this.settings = { + apis: { + web: { + url: 'http://web.example.com', + user: 'username', + pass: 'password' + } + } + }), + 'logger-sharelatex': (this.logger = { + log: sinon.stub(), + error: sinon.stub() + }) + } + })) + }) - return describe("joinProject", function() { - describe("successfully", function() { - beforeEach(function() { - this.response = { - project: { name: "Test project" }, - privilegeLevel: "owner", - isRestrictedUser: true - }; - this.request.post = sinon.stub().callsArgWith(1, null, {statusCode: 200}, this.response); - return this.WebApiManager.joinProject(this.project_id, this.user, this.callback); - }); + return describe('joinProject', function () { + describe('successfully', function () { + beforeEach(function () { + this.response = { + project: { name: 'Test project' }, + privilegeLevel: 'owner', + isRestrictedUser: true + } + this.request.post = sinon + .stub() + .callsArgWith(1, null, { statusCode: 200 }, this.response) + return this.WebApiManager.joinProject( + this.project_id, + this.user, + this.callback + ) + }) - it("should send a request to web to join the project", function() { - return this.request.post - .calledWith({ - url: `${this.settings.apis.web.url}/project/${this.project_id}/join`, - qs: { - user_id: this.user_id - }, - auth: { - user: this.settings.apis.web.user, - pass: this.settings.apis.web.pass, - sendImmediately: true - }, - json: true, - jar: false, - headers: {} - }) - .should.equal(true); - }); + it('should send a request to web to join the project', function () { + return this.request.post + .calledWith({ + url: `${this.settings.apis.web.url}/project/${this.project_id}/join`, + qs: { + user_id: this.user_id + }, + auth: { + user: this.settings.apis.web.user, + pass: this.settings.apis.web.pass, + sendImmediately: true + }, + json: true, + jar: false, + headers: {} + }) + .should.equal(true) + }) - return it("should return the project, privilegeLevel, and restricted flag", function() { - return this.callback - .calledWith(null, this.response.project, this.response.privilegeLevel, this.response.isRestrictedUser) - .should.equal(true); - }); - }); + return it('should return the project, privilegeLevel, and restricted flag', function () { + return this.callback + .calledWith( + null, + this.response.project, + this.response.privilegeLevel, + this.response.isRestrictedUser + ) + .should.equal(true) + }) + }) - describe("with an error from web", function() { - beforeEach(function() { - this.request.post = sinon.stub().callsArgWith(1, null, {statusCode: 500}, null); - return this.WebApiManager.joinProject(this.project_id, this.user_id, this.callback); - }); + describe('with an error from web', function () { + beforeEach(function () { + this.request.post = sinon + .stub() + .callsArgWith(1, null, { statusCode: 500 }, null) + return this.WebApiManager.joinProject( + this.project_id, + this.user_id, + this.callback + ) + }) - return it("should call the callback with an error", function() { - return this.callback - .calledWith(sinon.match({message: "non-success status code from web: 500"})) - .should.equal(true); - }); - }); + return it('should call the callback with an error', function () { + return this.callback + .calledWith( + sinon.match({ message: 'non-success status code from web: 500' }) + ) + .should.equal(true) + }) + }) - describe("with no data from web", function() { - beforeEach(function() { - this.request.post = sinon.stub().callsArgWith(1, null, {statusCode: 200}, null); - return this.WebApiManager.joinProject(this.project_id, this.user_id, this.callback); - }); + describe('with no data from web', function () { + beforeEach(function () { + this.request.post = sinon + .stub() + .callsArgWith(1, null, { statusCode: 200 }, null) + return this.WebApiManager.joinProject( + this.project_id, + this.user_id, + this.callback + ) + }) - return it("should call the callback with an error", function() { - return this.callback - .calledWith(sinon.match({message: "no data returned from joinProject request"})) - .should.equal(true); - }); - }); + return it('should call the callback with an error', function () { + return this.callback + .calledWith( + sinon.match({ + message: 'no data returned from joinProject request' + }) + ) + .should.equal(true) + }) + }) - return describe("when the project is over its rate limit", function() { - beforeEach(function() { - this.request.post = sinon.stub().callsArgWith(1, null, {statusCode: 429}, null); - return this.WebApiManager.joinProject(this.project_id, this.user_id, this.callback); - }); + return describe('when the project is over its rate limit', function () { + beforeEach(function () { + this.request.post = sinon + .stub() + .callsArgWith(1, null, { statusCode: 429 }, null) + return this.WebApiManager.joinProject( + this.project_id, + this.user_id, + this.callback + ) + }) - return it("should call the callback with a TooManyRequests error code", function() { - return this.callback - .calledWith(sinon.match({message: "rate-limit hit when joining project", code: "TooManyRequests"})) - .should.equal(true); - }); - }); - }); -}); + return it('should call the callback with a TooManyRequests error code', function () { + return this.callback + .calledWith( + sinon.match({ + message: 'rate-limit hit when joining project', + code: 'TooManyRequests' + }) + ) + .should.equal(true) + }) + }) + }) +}) diff --git a/services/real-time/test/unit/js/WebsocketControllerTests.js b/services/real-time/test/unit/js/WebsocketControllerTests.js index 58f417d1ca..8d8a39bb74 100644 --- a/services/real-time/test/unit/js/WebsocketControllerTests.js +++ b/services/real-time/test/unit/js/WebsocketControllerTests.js @@ -12,1085 +12,1483 @@ * DS102: Remove unnecessary code created because of implicit returns * Full docs: https://github.com/decaffeinate/decaffeinate/blob/master/docs/suggestions.md */ -const chai = require('chai'); -const should = chai.should(); -const sinon = require("sinon"); -const { - expect -} = chai; -const modulePath = "../../../app/js/WebsocketController.js"; -const SandboxedModule = require('sandboxed-module'); -const tk = require("timekeeper"); - -describe('WebsocketController', function() { - beforeEach(function() { - tk.freeze(new Date()); - this.project_id = "project-id-123"; - this.user = { - _id: (this.user_id = "user-id-123"), - first_name: "James", - last_name: "Allen", - email: "james@example.com", - signUpDate: new Date("2014-01-01"), - loginCount: 42 - }; - this.callback = sinon.stub(); - this.client = { - disconnected: false, - id: (this.client_id = "mock-client-id-123"), - publicId: `other-id-${Math.random()}`, - ol_context: {}, - join: sinon.stub(), - leave: sinon.stub() - }; - return this.WebsocketController = SandboxedModule.require(modulePath, { requires: { - "./WebApiManager": (this.WebApiManager = {}), - "./AuthorizationManager": (this.AuthorizationManager = {}), - "./DocumentUpdaterManager": (this.DocumentUpdaterManager = {}), - "./ConnectedUsersManager": (this.ConnectedUsersManager = {}), - "./WebsocketLoadBalancer": (this.WebsocketLoadBalancer = {}), - "logger-sharelatex": (this.logger = { log: sinon.stub(), error: sinon.stub(), warn: sinon.stub() }), - "metrics-sharelatex": (this.metrics = { - inc: sinon.stub(), - set: sinon.stub() - }), - "./RoomManager": (this.RoomManager = {}) - } - });}); - - afterEach(function() { return tk.reset(); }); - - describe("joinProject", function() { - describe("when authorised", function() { - beforeEach(function() { - this.client.id = "mock-client-id"; - this.project = { - name: "Test Project", - owner: { - _id: (this.owner_id = "mock-owner-id-123") - } - }; - this.privilegeLevel = "owner"; - this.ConnectedUsersManager.updateUserPosition = sinon.stub().callsArg(4); - this.isRestrictedUser = true; - this.WebApiManager.joinProject = sinon.stub().callsArgWith(2, null, this.project, this.privilegeLevel, this.isRestrictedUser); - this.RoomManager.joinProject = sinon.stub().callsArg(2); - return this.WebsocketController.joinProject(this.client, this.user, this.project_id, this.callback); - }); - - it("should load the project from web", function() { - return this.WebApiManager.joinProject - .calledWith(this.project_id, this.user) - .should.equal(true); - }); - - it("should join the project room", function() { - return this.RoomManager.joinProject.calledWith(this.client, this.project_id).should.equal(true); - }); - - it("should set the privilege level on the client", function() { - return this.client.ol_context.privilege_level.should.equal(this.privilegeLevel); - }); - it("should set the user's id on the client", function() { - return this.client.ol_context.user_id.should.equal(this.user._id); - }); - it("should set the user's email on the client", function() { - return this.client.ol_context.email.should.equal(this.user.email); - }); - it("should set the user's first_name on the client", function() { - return this.client.ol_context.first_name.should.equal(this.user.first_name); - }); - it("should set the user's last_name on the client", function() { - return this.client.ol_context.last_name.should.equal(this.user.last_name); - }); - it("should set the user's sign up date on the client", function() { - return this.client.ol_context.signup_date.should.equal(this.user.signUpDate); - }); - it("should set the user's login_count on the client", function() { - return this.client.ol_context.login_count.should.equal(this.user.loginCount); - }); - it("should set the connected time on the client", function() { - return this.client.ol_context.connected_time.should.equal(new Date()); - }); - it("should set the project_id on the client", function() { - return this.client.ol_context.project_id.should.equal(this.project_id); - }); - it("should set the project owner id on the client", function() { - return this.client.ol_context.owner_id.should.equal(this.owner_id); - }); - it("should set the is_restricted_user flag on the client", function() { - return this.client.ol_context.is_restricted_user.should.equal(this.isRestrictedUser); - }); - it("should call the callback with the project, privilegeLevel and protocolVersion", function() { - return this.callback - .calledWith(null, this.project, this.privilegeLevel, this.WebsocketController.PROTOCOL_VERSION) - .should.equal(true); - }); - - it("should mark the user as connected in ConnectedUsersManager", function() { - return this.ConnectedUsersManager.updateUserPosition - .calledWith(this.project_id, this.client.publicId, this.user, null) - .should.equal(true); - }); - - return it("should increment the join-project metric", function() { - return this.metrics.inc.calledWith("editor.join-project").should.equal(true); - }); - }); - - describe("when not authorized", function() { - beforeEach(function() { - this.WebApiManager.joinProject = sinon.stub().callsArgWith(2, null, null, null); - return this.WebsocketController.joinProject(this.client, this.user, this.project_id, this.callback); - }); - - it("should return an error", function() { - return this.callback - .calledWith(sinon.match({message: "not authorized"})) - .should.equal(true); - }); - - return it("should not log an error", function() { - return this.logger.error.called.should.equal(false); - }); - }); - - describe("when the subscribe failed", function() { - beforeEach(function() { - this.client.id = "mock-client-id"; - this.project = { - name: "Test Project", - owner: { - _id: (this.owner_id = "mock-owner-id-123") - } - }; - this.privilegeLevel = "owner"; - this.ConnectedUsersManager.updateUserPosition = sinon.stub().callsArg(4); - this.isRestrictedUser = true; - this.WebApiManager.joinProject = sinon.stub().callsArgWith(2, null, this.project, this.privilegeLevel, this.isRestrictedUser); - this.RoomManager.joinProject = sinon.stub().callsArgWith(2, new Error("subscribe failed")); - return this.WebsocketController.joinProject(this.client, this.user, this.project_id, this.callback); - }); - - return it("should return an error", function() { - this.callback - .calledWith(sinon.match({message: "subscribe failed"})) - .should.equal(true); - return this.callback.args[0][0].message.should.equal("subscribe failed"); - }); - }); - - describe("when the client has disconnected", function() { - beforeEach(function() { - this.client.disconnected = true; - this.WebApiManager.joinProject = sinon.stub().callsArg(2); - return this.WebsocketController.joinProject(this.client, this.user, this.project_id, this.callback); - }); - - it("should not call WebApiManager.joinProject", function() { - return expect(this.WebApiManager.joinProject.called).to.equal(false); - }); - - it("should call the callback with no details", function() { - return expect(this.callback.args[0]).to.deep.equal([]); - }); - - return it("should increment the editor.join-project.disconnected metric with a status", function() { - return expect(this.metrics.inc.calledWith('editor.join-project.disconnected', 1, {status: 'immediately'})).to.equal(true); - }); - }); - - return describe("when the client disconnects while WebApiManager.joinProject is running", function() { - beforeEach(function() { - this.WebApiManager.joinProject = (project, user, cb) => { - this.client.disconnected = true; - return cb(null, this.project, this.privilegeLevel, this.isRestrictedUser); - }; - - return this.WebsocketController.joinProject(this.client, this.user, this.project_id, this.callback); - }); - - it("should call the callback with no details", function() { - return expect(this.callback.args[0]).to.deep.equal([]); - }); - - return it("should increment the editor.join-project.disconnected metric with a status", function() { - return expect(this.metrics.inc.calledWith('editor.join-project.disconnected', 1, {status: 'after-web-api-call'})).to.equal(true); - }); - }); - }); - - describe("leaveProject", function() { - beforeEach(function() { - this.DocumentUpdaterManager.flushProjectToMongoAndDelete = sinon.stub().callsArg(1); - this.ConnectedUsersManager.markUserAsDisconnected = sinon.stub().callsArg(2); - this.WebsocketLoadBalancer.emitToRoom = sinon.stub(); - this.RoomManager.leaveProjectAndDocs = sinon.stub(); - this.clientsInRoom = []; - this.io = { - sockets: { - clients: room_id => { - if (room_id !== this.project_id) { - throw "expected room_id to be project_id"; - } - return this.clientsInRoom; - } - } - }; - this.client.ol_context.project_id = this.project_id; - this.client.ol_context.user_id = this.user_id; - this.WebsocketController.FLUSH_IF_EMPTY_DELAY = 0; - return tk.reset(); - }); // Allow setTimeout to work. - - describe("when the client did not joined a project yet", function() { - beforeEach(function(done) { - this.client.ol_context = {}; - return this.WebsocketController.leaveProject(this.io, this.client, done); - }); - - it("should bail out when calling leaveProject", function() { - this.WebsocketLoadBalancer.emitToRoom.called.should.equal(false); - this.RoomManager.leaveProjectAndDocs.called.should.equal(false); - return this.ConnectedUsersManager.markUserAsDisconnected.called.should.equal(false); - }); - - return it("should not inc any metric", function() { - return this.metrics.inc.called.should.equal(false); - }); - }); - - describe("when the project is empty", function() { - beforeEach(function(done) { - this.clientsInRoom = []; - return this.WebsocketController.leaveProject(this.io, this.client, done); - }); - - it("should end clientTracking.clientDisconnected to the project room", function() { - return this.WebsocketLoadBalancer.emitToRoom - .calledWith(this.project_id, "clientTracking.clientDisconnected", this.client.publicId) - .should.equal(true); - }); - - it("should mark the user as disconnected", function() { - return this.ConnectedUsersManager.markUserAsDisconnected - .calledWith(this.project_id, this.client.publicId) - .should.equal(true); - }); - - it("should flush the project in the document updater", function() { - return this.DocumentUpdaterManager.flushProjectToMongoAndDelete - .calledWith(this.project_id) - .should.equal(true); - }); - - it("should increment the leave-project metric", function() { - return this.metrics.inc.calledWith("editor.leave-project").should.equal(true); - }); - - return it("should track the disconnection in RoomManager", function() { - return this.RoomManager.leaveProjectAndDocs - .calledWith(this.client) - .should.equal(true); - }); - }); - - describe("when the project is not empty", function() { - beforeEach(function() { - this.clientsInRoom = ["mock-remaining-client"]; - return this.WebsocketController.leaveProject(this.io, this.client); - }); - - return it("should not flush the project in the document updater", function() { - return this.DocumentUpdaterManager.flushProjectToMongoAndDelete - .called.should.equal(false); - }); - }); - - describe("when client has not authenticated", function() { - beforeEach(function(done) { - this.client.ol_context.user_id = null; - this.client.ol_context.project_id = null; - return this.WebsocketController.leaveProject(this.io, this.client, done); - }); - - it("should not end clientTracking.clientDisconnected to the project room", function() { - return this.WebsocketLoadBalancer.emitToRoom - .calledWith(this.project_id, "clientTracking.clientDisconnected", this.client.publicId) - .should.equal(false); - }); - - it("should not mark the user as disconnected", function() { - return this.ConnectedUsersManager.markUserAsDisconnected - .calledWith(this.project_id, this.client.publicId) - .should.equal(false); - }); - - it("should not flush the project in the document updater", function() { - return this.DocumentUpdaterManager.flushProjectToMongoAndDelete - .calledWith(this.project_id) - .should.equal(false); - }); - - return it("should not increment the leave-project metric", function() { - return this.metrics.inc.calledWith("editor.leave-project").should.equal(false); - }); - }); - - return describe("when client has not joined a project", function() { - beforeEach(function(done) { - this.client.ol_context.user_id = this.user_id; - this.client.ol_context.project_id = null; - return this.WebsocketController.leaveProject(this.io, this.client, done); - }); - - it("should not end clientTracking.clientDisconnected to the project room", function() { - return this.WebsocketLoadBalancer.emitToRoom - .calledWith(this.project_id, "clientTracking.clientDisconnected", this.client.publicId) - .should.equal(false); - }); - - it("should not mark the user as disconnected", function() { - return this.ConnectedUsersManager.markUserAsDisconnected - .calledWith(this.project_id, this.client.publicId) - .should.equal(false); - }); - - it("should not flush the project in the document updater", function() { - return this.DocumentUpdaterManager.flushProjectToMongoAndDelete - .calledWith(this.project_id) - .should.equal(false); - }); - - return it("should not increment the leave-project metric", function() { - return this.metrics.inc.calledWith("editor.leave-project").should.equal(false); - }); - }); - }); - - describe("joinDoc", function() { - beforeEach(function() { - this.doc_id = "doc-id-123"; - this.doc_lines = ["doc", "lines"]; - this.version = 42; - this.ops = ["mock", "ops"]; - this.ranges = { "mock": "ranges" }; - this.options = {}; - - this.client.ol_context.project_id = this.project_id; - this.client.ol_context.is_restricted_user = false; - this.AuthorizationManager.addAccessToDoc = sinon.stub(); - this.AuthorizationManager.assertClientCanViewProject = sinon.stub().callsArgWith(1, null); - this.DocumentUpdaterManager.getDocument = sinon.stub().callsArgWith(3, null, this.doc_lines, this.version, this.ranges, this.ops); - return this.RoomManager.joinDoc = sinon.stub().callsArg(2); - }); - - describe("works", function() { - beforeEach(function() { - return this.WebsocketController.joinDoc(this.client, this.doc_id, -1, this.options, this.callback); - }); - - it("should check that the client is authorized to view the project", function() { - return this.AuthorizationManager.assertClientCanViewProject - .calledWith(this.client) - .should.equal(true); - }); - - it("should get the document from the DocumentUpdaterManager with fromVersion", function() { - return this.DocumentUpdaterManager.getDocument - .calledWith(this.project_id, this.doc_id, -1) - .should.equal(true); - }); - - it("should add permissions for the client to access the doc", function() { - return this.AuthorizationManager.addAccessToDoc - .calledWith(this.client, this.doc_id) - .should.equal(true); - }); - - it("should join the client to room for the doc_id", function() { - return this.RoomManager.joinDoc - .calledWith(this.client, this.doc_id) - .should.equal(true); - }); - - it("should call the callback with the lines, version, ranges and ops", function() { - return this.callback - .calledWith(null, this.doc_lines, this.version, this.ops, this.ranges) - .should.equal(true); - }); - - return it("should increment the join-doc metric", function() { - return this.metrics.inc.calledWith("editor.join-doc").should.equal(true); - }); - }); - - describe("with a fromVersion", function() { - beforeEach(function() { - this.fromVersion = 40; - return this.WebsocketController.joinDoc(this.client, this.doc_id, this.fromVersion, this.options, this.callback); - }); - - return it("should get the document from the DocumentUpdaterManager with fromVersion", function() { - return this.DocumentUpdaterManager.getDocument - .calledWith(this.project_id, this.doc_id, this.fromVersion) - .should.equal(true); - }); - }); - - describe("with doclines that need escaping", function() { - beforeEach(function() { - this.doc_lines.push(["räksmörgås"]); - return this.WebsocketController.joinDoc(this.client, this.doc_id, -1, this.options, this.callback); - }); - - return it("should call the callback with the escaped lines", function() { - const escaped_lines = this.callback.args[0][1]; - const escaped_word = escaped_lines.pop(); - escaped_word.should.equal('räksmörgÃ¥s'); - // Check that unescaping works - return decodeURIComponent(escape(escaped_word)).should.equal("räksmörgås"); - }); - }); - - describe("with comments that need encoding", function() { - beforeEach(function() { - this.ranges.comments = [{ op: { c: "räksmörgås" } }]; - return this.WebsocketController.joinDoc(this.client, this.doc_id, -1, { encodeRanges: true }, this.callback); - }); - - return it("should call the callback with the encoded comment", function() { - const encoded_comments = this.callback.args[0][4]; - const encoded_comment = encoded_comments.comments.pop(); - const encoded_comment_text = encoded_comment.op.c; - return encoded_comment_text.should.equal('räksmörgÃ¥s'); - }); - }); - - describe("with changes that need encoding", function() { - it("should call the callback with the encoded insert change", function() { - this.ranges.changes = [{ op: { i: "räksmörgås" } }]; - this.WebsocketController.joinDoc(this.client, this.doc_id, -1, { encodeRanges: true }, this.callback); - - const encoded_changes = this.callback.args[0][4]; - const encoded_change = encoded_changes.changes.pop(); - const encoded_change_text = encoded_change.op.i; - return encoded_change_text.should.equal('räksmörgÃ¥s'); - }); - - return it("should call the callback with the encoded delete change", function() { - this.ranges.changes = [{ op: { d: "räksmörgås" } }]; - this.WebsocketController.joinDoc(this.client, this.doc_id, -1, { encodeRanges: true }, this.callback); - - const encoded_changes = this.callback.args[0][4]; - const encoded_change = encoded_changes.changes.pop(); - const encoded_change_text = encoded_change.op.d; - return encoded_change_text.should.equal('räksmörgÃ¥s'); - }); - }); - - describe("when not authorized", function() { - beforeEach(function() { - this.AuthorizationManager.assertClientCanViewProject = sinon.stub().callsArgWith(1, (this.err = new Error("not authorized"))); - return this.WebsocketController.joinDoc(this.client, this.doc_id, -1, this.options, this.callback); - }); - - it("should call the callback with an error", function() { - return this.callback.calledWith(sinon.match({message: "not authorized"})).should.equal(true); - }); - - return it("should not call the DocumentUpdaterManager", function() { - return this.DocumentUpdaterManager.getDocument.called.should.equal(false); - }); - }); - - describe("with a restricted client", function() { - beforeEach(function() { - this.ranges.comments = [{op: {a: 1}}, {op: {a: 2}}]; - this.client.ol_context.is_restricted_user = true; - return this.WebsocketController.joinDoc(this.client, this.doc_id, -1, this.options, this.callback); - }); - - return it("should overwrite ranges.comments with an empty list", function() { - const ranges = this.callback.args[0][4]; - return expect(ranges.comments).to.deep.equal([]); - }); - }); - - describe("when the client has disconnected", function() { - beforeEach(function() { - this.client.disconnected = true; - return this.WebsocketController.joinDoc(this.client, this.doc_id, -1, this.options, this.callback); - }); - - it("should call the callback with no details", function() { - return expect(this.callback.args[0]).to.deep.equal([]); - }); - - it("should increment the editor.join-doc.disconnected metric with a status", function() { - return expect(this.metrics.inc.calledWith('editor.join-doc.disconnected', 1, {status: 'immediately'})).to.equal(true); - }); - - return it("should not get the document", function() { - return expect(this.DocumentUpdaterManager.getDocument.called).to.equal(false); - }); - }); - - describe("when the client disconnects while RoomManager.joinDoc is running", function() { - beforeEach(function() { - this.RoomManager.joinDoc = (client, doc_id, cb) => { - this.client.disconnected = true; - return cb(); - }; - - return this.WebsocketController.joinDoc(this.client, this.doc_id, -1, this.options, this.callback); - }); - - it("should call the callback with no details", function() { - return expect(this.callback.args[0]).to.deep.equal([]); - }); - - it("should increment the editor.join-doc.disconnected metric with a status", function() { - return expect(this.metrics.inc.calledWith('editor.join-doc.disconnected', 1, {status: 'after-joining-room'})).to.equal(true); - }); - - return it("should not get the document", function() { - return expect(this.DocumentUpdaterManager.getDocument.called).to.equal(false); - }); - }); - - return describe("when the client disconnects while DocumentUpdaterManager.getDocument is running", function() { - beforeEach(function() { - this.DocumentUpdaterManager.getDocument = (project_id, doc_id, fromVersion, callback) => { - this.client.disconnected = true; - return callback(null, this.doc_lines, this.version, this.ranges, this.ops); - }; - - return this.WebsocketController.joinDoc(this.client, this.doc_id, -1, this.options, this.callback); - }); - - it("should call the callback with no details", function() { - return expect(this.callback.args[0]).to.deep.equal([]); - }); - - return it("should increment the editor.join-doc.disconnected metric with a status", function() { - return expect(this.metrics.inc.calledWith('editor.join-doc.disconnected', 1, {status: 'after-doc-updater-call'})).to.equal(true); - }); - }); - }); - - describe("leaveDoc", function() { - beforeEach(function() { - this.doc_id = "doc-id-123"; - this.client.ol_context.project_id = this.project_id; - this.RoomManager.leaveDoc = sinon.stub(); - return this.WebsocketController.leaveDoc(this.client, this.doc_id, this.callback); - }); - - it("should remove the client from the doc_id room", function() { - return this.RoomManager.leaveDoc - .calledWith(this.client, this.doc_id).should.equal(true); - }); - - it("should call the callback", function() { - return this.callback.called.should.equal(true); - }); - - return it("should increment the leave-doc metric", function() { - return this.metrics.inc.calledWith("editor.leave-doc").should.equal(true); - }); - }); - - describe("getConnectedUsers", function() { - beforeEach(function() { - this.client.ol_context.project_id = this.project_id; - this.users = ["mock", "users"]; - this.WebsocketLoadBalancer.emitToRoom = sinon.stub(); - return this.ConnectedUsersManager.getConnectedUsers = sinon.stub().callsArgWith(1, null, this.users); - }); - - describe("when authorized", function() { - beforeEach(function(done) { - this.AuthorizationManager.assertClientCanViewProject = sinon.stub().callsArgWith(1, null); - return this.WebsocketController.getConnectedUsers(this.client, (...args) => { - this.callback(...Array.from(args || [])); - return done(); - }); - }); - - it("should check that the client is authorized to view the project", function() { - return this.AuthorizationManager.assertClientCanViewProject - .calledWith(this.client) - .should.equal(true); - }); - - it("should broadcast a request to update the client list", function() { - return this.WebsocketLoadBalancer.emitToRoom - .calledWith(this.project_id, "clientTracking.refresh") - .should.equal(true); - }); - - it("should get the connected users for the project", function() { - return this.ConnectedUsersManager.getConnectedUsers - .calledWith(this.project_id) - .should.equal(true); - }); - - it("should return the users", function() { - return this.callback.calledWith(null, this.users).should.equal(true); - }); - - return it("should increment the get-connected-users metric", function() { - return this.metrics.inc.calledWith("editor.get-connected-users").should.equal(true); - }); - }); - - describe("when not authorized", function() { - beforeEach(function() { - this.AuthorizationManager.assertClientCanViewProject = sinon.stub().callsArgWith(1, (this.err = new Error("not authorized"))); - return this.WebsocketController.getConnectedUsers(this.client, this.callback); - }); - - it("should not get the connected users for the project", function() { - return this.ConnectedUsersManager.getConnectedUsers - .called - .should.equal(false); - }); - - return it("should return an error", function() { - return this.callback.calledWith(this.err).should.equal(true); - }); - }); - - describe("when restricted user", function() { - beforeEach(function() { - this.client.ol_context.is_restricted_user = true; - this.AuthorizationManager.assertClientCanViewProject = sinon.stub().callsArgWith(1, null); - return this.WebsocketController.getConnectedUsers(this.client, this.callback); - }); - - it("should return an empty array of users", function() { - return this.callback.calledWith(null, []).should.equal(true); - }); - - return it("should not get the connected users for the project", function() { - return this.ConnectedUsersManager.getConnectedUsers - .called - .should.equal(false); - }); - }); - - return describe("when the client has disconnected", function() { - beforeEach(function() { - this.client.disconnected = true; - this.AuthorizationManager.assertClientCanViewProject = sinon.stub(); - return this.WebsocketController.getConnectedUsers(this.client, this.callback); - }); - - it("should call the callback with no details", function() { - return expect(this.callback.args[0]).to.deep.equal([]); - }); - - return it("should not check permissions", function() { - return expect(this.AuthorizationManager.assertClientCanViewProject.called).to.equal(false); - }); - }); - }); - - describe("updateClientPosition", function() { - beforeEach(function() { - this.WebsocketLoadBalancer.emitToRoom = sinon.stub(); - this.ConnectedUsersManager.updateUserPosition = sinon.stub().callsArgWith(4); - this.AuthorizationManager.assertClientCanViewProjectAndDoc = sinon.stub().callsArgWith(2, null); - return this.update = { - doc_id: (this.doc_id = "doc-id-123"), - row: (this.row = 42), - column: (this.column = 37) - };}); - - describe("with a logged in user", function() { - beforeEach(function() { - this.client.ol_context = { - project_id: this.project_id, - first_name: (this.first_name = "Douglas"), - last_name: (this.last_name = "Adams"), - email: (this.email = "joe@example.com"), - user_id: (this.user_id = "user-id-123") - }; - this.WebsocketController.updateClientPosition(this.client, this.update); - - return this.populatedCursorData = { - doc_id: this.doc_id, - id: this.client.publicId, - name: `${this.first_name} ${this.last_name}`, - row: this.row, - column: this.column, - email: this.email, - user_id: this.user_id - }; - }); - - it("should send the update to the project room with the user's name", function() { - return this.WebsocketLoadBalancer.emitToRoom.calledWith(this.project_id, "clientTracking.clientUpdated", this.populatedCursorData).should.equal(true); - }); - - it("should send the cursor data to the connected user manager", function(done){ - this.ConnectedUsersManager.updateUserPosition.calledWith(this.project_id, this.client.publicId, { - _id: this.user_id, - email: this.email, - first_name: this.first_name, - last_name: this.last_name - }, { - row: this.row, - column: this.column, - doc_id: this.doc_id - }).should.equal(true); - return done(); - }); - - return it("should increment the update-client-position metric at 0.1 frequency", function() { - return this.metrics.inc.calledWith("editor.update-client-position", 0.1).should.equal(true); - }); - }); - - describe("with a logged in user who has no last_name set", function() { - beforeEach(function() { - this.client.ol_context = { - project_id: this.project_id, - first_name: (this.first_name = "Douglas"), - last_name: undefined, - email: (this.email = "joe@example.com"), - user_id: (this.user_id = "user-id-123") - }; - this.WebsocketController.updateClientPosition(this.client, this.update); - - return this.populatedCursorData = { - doc_id: this.doc_id, - id: this.client.publicId, - name: `${this.first_name}`, - row: this.row, - column: this.column, - email: this.email, - user_id: this.user_id - }; - }); - - it("should send the update to the project room with the user's name", function() { - return this.WebsocketLoadBalancer.emitToRoom.calledWith(this.project_id, "clientTracking.clientUpdated", this.populatedCursorData).should.equal(true); - }); - - it("should send the cursor data to the connected user manager", function(done){ - this.ConnectedUsersManager.updateUserPosition.calledWith(this.project_id, this.client.publicId, { - _id: this.user_id, - email: this.email, - first_name: this.first_name, - last_name: undefined - }, { - row: this.row, - column: this.column, - doc_id: this.doc_id - }).should.equal(true); - return done(); - }); - - return it("should increment the update-client-position metric at 0.1 frequency", function() { - return this.metrics.inc.calledWith("editor.update-client-position", 0.1).should.equal(true); - }); - }); - - describe("with a logged in user who has no first_name set", function() { - beforeEach(function() { - this.client.ol_context = { - project_id: this.project_id, - first_name: undefined, - last_name: (this.last_name = "Adams"), - email: (this.email = "joe@example.com"), - user_id: (this.user_id = "user-id-123") - }; - this.WebsocketController.updateClientPosition(this.client, this.update); - - return this.populatedCursorData = { - doc_id: this.doc_id, - id: this.client.publicId, - name: `${this.last_name}`, - row: this.row, - column: this.column, - email: this.email, - user_id: this.user_id - }; - }); - - it("should send the update to the project room with the user's name", function() { - return this.WebsocketLoadBalancer.emitToRoom.calledWith(this.project_id, "clientTracking.clientUpdated", this.populatedCursorData).should.equal(true); - }); - - it("should send the cursor data to the connected user manager", function(done){ - this.ConnectedUsersManager.updateUserPosition.calledWith(this.project_id, this.client.publicId, { - _id: this.user_id, - email: this.email, - first_name: undefined, - last_name: this.last_name - }, { - row: this.row, - column: this.column, - doc_id: this.doc_id - }).should.equal(true); - return done(); - }); - - return it("should increment the update-client-position metric at 0.1 frequency", function() { - return this.metrics.inc.calledWith("editor.update-client-position", 0.1).should.equal(true); - }); - }); - describe("with a logged in user who has no names set", function() { - beforeEach(function() { - this.client.ol_context = { - project_id: this.project_id, - first_name: undefined, - last_name: undefined, - email: (this.email = "joe@example.com"), - user_id: (this.user_id = "user-id-123") - }; - return this.WebsocketController.updateClientPosition(this.client, this.update); - }); - - return it("should send the update to the project name with no name", function() { - return this.WebsocketLoadBalancer.emitToRoom - .calledWith(this.project_id, "clientTracking.clientUpdated", { - doc_id: this.doc_id, - id: this.client.publicId, - user_id: this.user_id, - name: "", - row: this.row, - column: this.column, - email: this.email - }) - .should.equal(true); - }); - }); - - - describe("with an anonymous user", function() { - beforeEach(function() { - this.client.ol_context = { - project_id: this.project_id - }; - return this.WebsocketController.updateClientPosition(this.client, this.update); - }); - - it("should send the update to the project room with no name", function() { - return this.WebsocketLoadBalancer.emitToRoom - .calledWith(this.project_id, "clientTracking.clientUpdated", { - doc_id: this.doc_id, - id: this.client.publicId, - name: "", - row: this.row, - column: this.column - }) - .should.equal(true); - }); - - return it("should not send cursor data to the connected user manager", function(done){ - this.ConnectedUsersManager.updateUserPosition.called.should.equal(false); - return done(); - }); - }); - - return describe("when the client has disconnected", function() { - beforeEach(function() { - this.client.disconnected = true; - this.AuthorizationManager.assertClientCanViewProjectAndDoc = sinon.stub(); - return this.WebsocketController.updateClientPosition(this.client, this.update, this.callback); - }); - - it("should call the callback with no details", function() { - return expect(this.callback.args[0]).to.deep.equal([]); - }); - - return it("should not check permissions", function() { - return expect(this.AuthorizationManager.assertClientCanViewProjectAndDoc.called).to.equal(false); - }); - }); - }); - - describe("applyOtUpdate", function() { - beforeEach(function() { - this.update = {op: {p: 12, t: "foo"}}; - this.client.ol_context.user_id = this.user_id; - this.client.ol_context.project_id = this.project_id; - this.WebsocketController._assertClientCanApplyUpdate = sinon.stub().yields(); - return this.DocumentUpdaterManager.queueChange = sinon.stub().callsArg(3); - }); - - describe("succesfully", function() { - beforeEach(function() { - return this.WebsocketController.applyOtUpdate(this.client, this.doc_id, this.update, this.callback); - }); - - it("should set the source of the update to the client id", function() { - return this.update.meta.source.should.equal(this.client.publicId); - }); - - it("should set the user_id of the update to the user id", function() { - return this.update.meta.user_id.should.equal(this.user_id); - }); - - it("should queue the update", function() { - return this.DocumentUpdaterManager.queueChange - .calledWith(this.project_id, this.doc_id, this.update) - .should.equal(true); - }); - - it("should call the callback", function() { - return this.callback.called.should.equal(true); - }); - - return it("should increment the doc updates", function() { - return this.metrics.inc.calledWith("editor.doc-update").should.equal(true); - }); - }); - - describe("unsuccessfully", function() { - beforeEach(function() { - this.client.disconnect = sinon.stub(); - this.DocumentUpdaterManager.queueChange = sinon.stub().callsArgWith(3, (this.error = new Error("Something went wrong"))); - return this.WebsocketController.applyOtUpdate(this.client, this.doc_id, this.update, this.callback); - }); - - it("should disconnect the client", function() { - return this.client.disconnect.called.should.equal(true); - }); - - it("should log an error", function() { - return this.logger.error.called.should.equal(true); - }); - - return it("should call the callback with the error", function() { - return this.callback.calledWith(this.error).should.equal(true); - }); - }); - - describe("when not authorized", function() { - beforeEach(function() { - this.client.disconnect = sinon.stub(); - this.WebsocketController._assertClientCanApplyUpdate = sinon.stub().yields(this.error = new Error("not authorized")); - return this.WebsocketController.applyOtUpdate(this.client, this.doc_id, this.update, this.callback); - }); - - // This happens in a setTimeout to allow the client a chance to receive the error first. - // I'm not sure how to unit test, but it is acceptance tested. - // it "should disconnect the client", -> - // @client.disconnect.called.should.equal true - - it("should log a warning", function() { - return this.logger.warn.called.should.equal(true); - }); - - return it("should call the callback with the error", function() { - return this.callback.calledWith(this.error).should.equal(true); - }); - }); - - return describe("update_too_large", function() { - beforeEach(function(done) { - this.client.disconnect = sinon.stub(); - this.client.emit = sinon.stub(); - this.client.ol_context.user_id = this.user_id; - this.client.ol_context.project_id = this.project_id; - const error = new Error("update is too large"); - error.updateSize = 7372835; - this.DocumentUpdaterManager.queueChange = sinon.stub().callsArgWith(3, error); - this.WebsocketController.applyOtUpdate(this.client, this.doc_id, this.update, this.callback); - return setTimeout(() => done() - , 1); - }); - - it("should call the callback with no error", function() { - this.callback.called.should.equal(true); - return this.callback.args[0].should.deep.equal([]); - }); - - it("should log a warning with the size and context", function() { - this.logger.warn.called.should.equal(true); - return this.logger.warn.args[0].should.deep.equal([{ - user_id: this.user_id, project_id: this.project_id, doc_id: this.doc_id, updateSize: 7372835 - }, 'update is too large']); - }); - - describe("after 100ms", function() { - beforeEach(function(done) { return setTimeout(done, 100); }); - - it("should send an otUpdateError the client", function() { - return this.client.emit.calledWith('otUpdateError').should.equal(true); - }); - - return it("should disconnect the client", function() { - return this.client.disconnect.called.should.equal(true); - }); - }); - - return describe("when the client disconnects during the next 100ms", function() { - beforeEach(function(done) { - this.client.disconnected = true; - return setTimeout(done, 100); - }); - - it("should not send an otUpdateError the client", function() { - return this.client.emit.calledWith('otUpdateError').should.equal(false); - }); - - it("should not disconnect the client", function() { - return this.client.disconnect.called.should.equal(false); - }); - - return it("should increment the editor.doc-update.disconnected metric with a status", function() { - return expect(this.metrics.inc.calledWith('editor.doc-update.disconnected', 1, {status:'at-otUpdateError'})).to.equal(true); - }); - }); - }); - }); - - return describe("_assertClientCanApplyUpdate", function() { - beforeEach(function() { - this.edit_update = { op: [{i: "foo", p: 42}, {c: "bar", p: 132}] }; // comments may still be in an edit op - this.comment_update = { op: [{c: "bar", p: 132}] }; - this.AuthorizationManager.assertClientCanEditProjectAndDoc = sinon.stub(); - return this.AuthorizationManager.assertClientCanViewProjectAndDoc = sinon.stub(); - }); - - describe("with a read-write client", function() { return it("should return successfully", function(done) { - this.AuthorizationManager.assertClientCanEditProjectAndDoc.yields(null); - return this.WebsocketController._assertClientCanApplyUpdate(this.client, this.doc_id, this.edit_update, (error) => { - expect(error).to.be.null; - return done(); - }); - }); }); - - describe("with a read-only client and an edit op", function() { return it("should return an error", function(done) { - this.AuthorizationManager.assertClientCanEditProjectAndDoc.yields(new Error("not authorized")); - this.AuthorizationManager.assertClientCanViewProjectAndDoc.yields(null); - return this.WebsocketController._assertClientCanApplyUpdate(this.client, this.doc_id, this.edit_update, (error) => { - expect(error.message).to.equal("not authorized"); - return done(); - }); - }); }); - - describe("with a read-only client and a comment op", function() { return it("should return successfully", function(done) { - this.AuthorizationManager.assertClientCanEditProjectAndDoc.yields(new Error("not authorized")); - this.AuthorizationManager.assertClientCanViewProjectAndDoc.yields(null); - return this.WebsocketController._assertClientCanApplyUpdate(this.client, this.doc_id, this.comment_update, (error) => { - expect(error).to.be.null; - return done(); - }); - }); }); - - return describe("with a totally unauthorized client", function() { return it("should return an error", function(done) { - this.AuthorizationManager.assertClientCanEditProjectAndDoc.yields(new Error("not authorized")); - this.AuthorizationManager.assertClientCanViewProjectAndDoc.yields(new Error("not authorized")); - return this.WebsocketController._assertClientCanApplyUpdate(this.client, this.doc_id, this.comment_update, (error) => { - expect(error.message).to.equal("not authorized"); - return done(); - }); - }); }); - }); -}); +const chai = require('chai') +const should = chai.should() +const sinon = require('sinon') +const { expect } = chai +const modulePath = '../../../app/js/WebsocketController.js' +const SandboxedModule = require('sandboxed-module') +const tk = require('timekeeper') + +describe('WebsocketController', function () { + beforeEach(function () { + tk.freeze(new Date()) + this.project_id = 'project-id-123' + this.user = { + _id: (this.user_id = 'user-id-123'), + first_name: 'James', + last_name: 'Allen', + email: 'james@example.com', + signUpDate: new Date('2014-01-01'), + loginCount: 42 + } + this.callback = sinon.stub() + this.client = { + disconnected: false, + id: (this.client_id = 'mock-client-id-123'), + publicId: `other-id-${Math.random()}`, + ol_context: {}, + join: sinon.stub(), + leave: sinon.stub() + } + return (this.WebsocketController = SandboxedModule.require(modulePath, { + requires: { + './WebApiManager': (this.WebApiManager = {}), + './AuthorizationManager': (this.AuthorizationManager = {}), + './DocumentUpdaterManager': (this.DocumentUpdaterManager = {}), + './ConnectedUsersManager': (this.ConnectedUsersManager = {}), + './WebsocketLoadBalancer': (this.WebsocketLoadBalancer = {}), + 'logger-sharelatex': (this.logger = { + log: sinon.stub(), + error: sinon.stub(), + warn: sinon.stub() + }), + 'metrics-sharelatex': (this.metrics = { + inc: sinon.stub(), + set: sinon.stub() + }), + './RoomManager': (this.RoomManager = {}) + } + })) + }) + + afterEach(function () { + return tk.reset() + }) + + describe('joinProject', function () { + describe('when authorised', function () { + beforeEach(function () { + this.client.id = 'mock-client-id' + this.project = { + name: 'Test Project', + owner: { + _id: (this.owner_id = 'mock-owner-id-123') + } + } + this.privilegeLevel = 'owner' + this.ConnectedUsersManager.updateUserPosition = sinon.stub().callsArg(4) + this.isRestrictedUser = true + this.WebApiManager.joinProject = sinon + .stub() + .callsArgWith( + 2, + null, + this.project, + this.privilegeLevel, + this.isRestrictedUser + ) + this.RoomManager.joinProject = sinon.stub().callsArg(2) + return this.WebsocketController.joinProject( + this.client, + this.user, + this.project_id, + this.callback + ) + }) + + it('should load the project from web', function () { + return this.WebApiManager.joinProject + .calledWith(this.project_id, this.user) + .should.equal(true) + }) + + it('should join the project room', function () { + return this.RoomManager.joinProject + .calledWith(this.client, this.project_id) + .should.equal(true) + }) + + it('should set the privilege level on the client', function () { + return this.client.ol_context.privilege_level.should.equal( + this.privilegeLevel + ) + }) + it("should set the user's id on the client", function () { + return this.client.ol_context.user_id.should.equal(this.user._id) + }) + it("should set the user's email on the client", function () { + return this.client.ol_context.email.should.equal(this.user.email) + }) + it("should set the user's first_name on the client", function () { + return this.client.ol_context.first_name.should.equal( + this.user.first_name + ) + }) + it("should set the user's last_name on the client", function () { + return this.client.ol_context.last_name.should.equal( + this.user.last_name + ) + }) + it("should set the user's sign up date on the client", function () { + return this.client.ol_context.signup_date.should.equal( + this.user.signUpDate + ) + }) + it("should set the user's login_count on the client", function () { + return this.client.ol_context.login_count.should.equal( + this.user.loginCount + ) + }) + it('should set the connected time on the client', function () { + return this.client.ol_context.connected_time.should.equal(new Date()) + }) + it('should set the project_id on the client', function () { + return this.client.ol_context.project_id.should.equal(this.project_id) + }) + it('should set the project owner id on the client', function () { + return this.client.ol_context.owner_id.should.equal(this.owner_id) + }) + it('should set the is_restricted_user flag on the client', function () { + return this.client.ol_context.is_restricted_user.should.equal( + this.isRestrictedUser + ) + }) + it('should call the callback with the project, privilegeLevel and protocolVersion', function () { + return this.callback + .calledWith( + null, + this.project, + this.privilegeLevel, + this.WebsocketController.PROTOCOL_VERSION + ) + .should.equal(true) + }) + + it('should mark the user as connected in ConnectedUsersManager', function () { + return this.ConnectedUsersManager.updateUserPosition + .calledWith(this.project_id, this.client.publicId, this.user, null) + .should.equal(true) + }) + + return it('should increment the join-project metric', function () { + return this.metrics.inc + .calledWith('editor.join-project') + .should.equal(true) + }) + }) + + describe('when not authorized', function () { + beforeEach(function () { + this.WebApiManager.joinProject = sinon + .stub() + .callsArgWith(2, null, null, null) + return this.WebsocketController.joinProject( + this.client, + this.user, + this.project_id, + this.callback + ) + }) + + it('should return an error', function () { + return this.callback + .calledWith(sinon.match({ message: 'not authorized' })) + .should.equal(true) + }) + + return it('should not log an error', function () { + return this.logger.error.called.should.equal(false) + }) + }) + + describe('when the subscribe failed', function () { + beforeEach(function () { + this.client.id = 'mock-client-id' + this.project = { + name: 'Test Project', + owner: { + _id: (this.owner_id = 'mock-owner-id-123') + } + } + this.privilegeLevel = 'owner' + this.ConnectedUsersManager.updateUserPosition = sinon.stub().callsArg(4) + this.isRestrictedUser = true + this.WebApiManager.joinProject = sinon + .stub() + .callsArgWith( + 2, + null, + this.project, + this.privilegeLevel, + this.isRestrictedUser + ) + this.RoomManager.joinProject = sinon + .stub() + .callsArgWith(2, new Error('subscribe failed')) + return this.WebsocketController.joinProject( + this.client, + this.user, + this.project_id, + this.callback + ) + }) + + return it('should return an error', function () { + this.callback + .calledWith(sinon.match({ message: 'subscribe failed' })) + .should.equal(true) + return this.callback.args[0][0].message.should.equal('subscribe failed') + }) + }) + + describe('when the client has disconnected', function () { + beforeEach(function () { + this.client.disconnected = true + this.WebApiManager.joinProject = sinon.stub().callsArg(2) + return this.WebsocketController.joinProject( + this.client, + this.user, + this.project_id, + this.callback + ) + }) + + it('should not call WebApiManager.joinProject', function () { + return expect(this.WebApiManager.joinProject.called).to.equal(false) + }) + + it('should call the callback with no details', function () { + return expect(this.callback.args[0]).to.deep.equal([]) + }) + + return it('should increment the editor.join-project.disconnected metric with a status', function () { + return expect( + this.metrics.inc.calledWith('editor.join-project.disconnected', 1, { + status: 'immediately' + }) + ).to.equal(true) + }) + }) + + return describe('when the client disconnects while WebApiManager.joinProject is running', function () { + beforeEach(function () { + this.WebApiManager.joinProject = (project, user, cb) => { + this.client.disconnected = true + return cb( + null, + this.project, + this.privilegeLevel, + this.isRestrictedUser + ) + } + + return this.WebsocketController.joinProject( + this.client, + this.user, + this.project_id, + this.callback + ) + }) + + it('should call the callback with no details', function () { + return expect(this.callback.args[0]).to.deep.equal([]) + }) + + return it('should increment the editor.join-project.disconnected metric with a status', function () { + return expect( + this.metrics.inc.calledWith('editor.join-project.disconnected', 1, { + status: 'after-web-api-call' + }) + ).to.equal(true) + }) + }) + }) + + describe('leaveProject', function () { + beforeEach(function () { + this.DocumentUpdaterManager.flushProjectToMongoAndDelete = sinon + .stub() + .callsArg(1) + this.ConnectedUsersManager.markUserAsDisconnected = sinon + .stub() + .callsArg(2) + this.WebsocketLoadBalancer.emitToRoom = sinon.stub() + this.RoomManager.leaveProjectAndDocs = sinon.stub() + this.clientsInRoom = [] + this.io = { + sockets: { + clients: (room_id) => { + if (room_id !== this.project_id) { + throw 'expected room_id to be project_id' + } + return this.clientsInRoom + } + } + } + this.client.ol_context.project_id = this.project_id + this.client.ol_context.user_id = this.user_id + this.WebsocketController.FLUSH_IF_EMPTY_DELAY = 0 + return tk.reset() + }) // Allow setTimeout to work. + + describe('when the client did not joined a project yet', function () { + beforeEach(function (done) { + this.client.ol_context = {} + return this.WebsocketController.leaveProject(this.io, this.client, done) + }) + + it('should bail out when calling leaveProject', function () { + this.WebsocketLoadBalancer.emitToRoom.called.should.equal(false) + this.RoomManager.leaveProjectAndDocs.called.should.equal(false) + return this.ConnectedUsersManager.markUserAsDisconnected.called.should.equal( + false + ) + }) + + return it('should not inc any metric', function () { + return this.metrics.inc.called.should.equal(false) + }) + }) + + describe('when the project is empty', function () { + beforeEach(function (done) { + this.clientsInRoom = [] + return this.WebsocketController.leaveProject(this.io, this.client, done) + }) + + it('should end clientTracking.clientDisconnected to the project room', function () { + return this.WebsocketLoadBalancer.emitToRoom + .calledWith( + this.project_id, + 'clientTracking.clientDisconnected', + this.client.publicId + ) + .should.equal(true) + }) + + it('should mark the user as disconnected', function () { + return this.ConnectedUsersManager.markUserAsDisconnected + .calledWith(this.project_id, this.client.publicId) + .should.equal(true) + }) + + it('should flush the project in the document updater', function () { + return this.DocumentUpdaterManager.flushProjectToMongoAndDelete + .calledWith(this.project_id) + .should.equal(true) + }) + + it('should increment the leave-project metric', function () { + return this.metrics.inc + .calledWith('editor.leave-project') + .should.equal(true) + }) + + return it('should track the disconnection in RoomManager', function () { + return this.RoomManager.leaveProjectAndDocs + .calledWith(this.client) + .should.equal(true) + }) + }) + + describe('when the project is not empty', function () { + beforeEach(function () { + this.clientsInRoom = ['mock-remaining-client'] + return this.WebsocketController.leaveProject(this.io, this.client) + }) + + return it('should not flush the project in the document updater', function () { + return this.DocumentUpdaterManager.flushProjectToMongoAndDelete.called.should.equal( + false + ) + }) + }) + + describe('when client has not authenticated', function () { + beforeEach(function (done) { + this.client.ol_context.user_id = null + this.client.ol_context.project_id = null + return this.WebsocketController.leaveProject(this.io, this.client, done) + }) + + it('should not end clientTracking.clientDisconnected to the project room', function () { + return this.WebsocketLoadBalancer.emitToRoom + .calledWith( + this.project_id, + 'clientTracking.clientDisconnected', + this.client.publicId + ) + .should.equal(false) + }) + + it('should not mark the user as disconnected', function () { + return this.ConnectedUsersManager.markUserAsDisconnected + .calledWith(this.project_id, this.client.publicId) + .should.equal(false) + }) + + it('should not flush the project in the document updater', function () { + return this.DocumentUpdaterManager.flushProjectToMongoAndDelete + .calledWith(this.project_id) + .should.equal(false) + }) + + return it('should not increment the leave-project metric', function () { + return this.metrics.inc + .calledWith('editor.leave-project') + .should.equal(false) + }) + }) + + return describe('when client has not joined a project', function () { + beforeEach(function (done) { + this.client.ol_context.user_id = this.user_id + this.client.ol_context.project_id = null + return this.WebsocketController.leaveProject(this.io, this.client, done) + }) + + it('should not end clientTracking.clientDisconnected to the project room', function () { + return this.WebsocketLoadBalancer.emitToRoom + .calledWith( + this.project_id, + 'clientTracking.clientDisconnected', + this.client.publicId + ) + .should.equal(false) + }) + + it('should not mark the user as disconnected', function () { + return this.ConnectedUsersManager.markUserAsDisconnected + .calledWith(this.project_id, this.client.publicId) + .should.equal(false) + }) + + it('should not flush the project in the document updater', function () { + return this.DocumentUpdaterManager.flushProjectToMongoAndDelete + .calledWith(this.project_id) + .should.equal(false) + }) + + return it('should not increment the leave-project metric', function () { + return this.metrics.inc + .calledWith('editor.leave-project') + .should.equal(false) + }) + }) + }) + + describe('joinDoc', function () { + beforeEach(function () { + this.doc_id = 'doc-id-123' + this.doc_lines = ['doc', 'lines'] + this.version = 42 + this.ops = ['mock', 'ops'] + this.ranges = { mock: 'ranges' } + this.options = {} + + this.client.ol_context.project_id = this.project_id + this.client.ol_context.is_restricted_user = false + this.AuthorizationManager.addAccessToDoc = sinon.stub() + this.AuthorizationManager.assertClientCanViewProject = sinon + .stub() + .callsArgWith(1, null) + this.DocumentUpdaterManager.getDocument = sinon + .stub() + .callsArgWith( + 3, + null, + this.doc_lines, + this.version, + this.ranges, + this.ops + ) + return (this.RoomManager.joinDoc = sinon.stub().callsArg(2)) + }) + + describe('works', function () { + beforeEach(function () { + return this.WebsocketController.joinDoc( + this.client, + this.doc_id, + -1, + this.options, + this.callback + ) + }) + + it('should check that the client is authorized to view the project', function () { + return this.AuthorizationManager.assertClientCanViewProject + .calledWith(this.client) + .should.equal(true) + }) + + it('should get the document from the DocumentUpdaterManager with fromVersion', function () { + return this.DocumentUpdaterManager.getDocument + .calledWith(this.project_id, this.doc_id, -1) + .should.equal(true) + }) + + it('should add permissions for the client to access the doc', function () { + return this.AuthorizationManager.addAccessToDoc + .calledWith(this.client, this.doc_id) + .should.equal(true) + }) + + it('should join the client to room for the doc_id', function () { + return this.RoomManager.joinDoc + .calledWith(this.client, this.doc_id) + .should.equal(true) + }) + + it('should call the callback with the lines, version, ranges and ops', function () { + return this.callback + .calledWith(null, this.doc_lines, this.version, this.ops, this.ranges) + .should.equal(true) + }) + + return it('should increment the join-doc metric', function () { + return this.metrics.inc.calledWith('editor.join-doc').should.equal(true) + }) + }) + + describe('with a fromVersion', function () { + beforeEach(function () { + this.fromVersion = 40 + return this.WebsocketController.joinDoc( + this.client, + this.doc_id, + this.fromVersion, + this.options, + this.callback + ) + }) + + return it('should get the document from the DocumentUpdaterManager with fromVersion', function () { + return this.DocumentUpdaterManager.getDocument + .calledWith(this.project_id, this.doc_id, this.fromVersion) + .should.equal(true) + }) + }) + + describe('with doclines that need escaping', function () { + beforeEach(function () { + this.doc_lines.push(['räksmörgås']) + return this.WebsocketController.joinDoc( + this.client, + this.doc_id, + -1, + this.options, + this.callback + ) + }) + + return it('should call the callback with the escaped lines', function () { + const escaped_lines = this.callback.args[0][1] + const escaped_word = escaped_lines.pop() + escaped_word.should.equal('räksmörgÃ¥s') + // Check that unescaping works + return decodeURIComponent(escape(escaped_word)).should.equal( + 'räksmörgås' + ) + }) + }) + + describe('with comments that need encoding', function () { + beforeEach(function () { + this.ranges.comments = [{ op: { c: 'räksmörgås' } }] + return this.WebsocketController.joinDoc( + this.client, + this.doc_id, + -1, + { encodeRanges: true }, + this.callback + ) + }) + + return it('should call the callback with the encoded comment', function () { + const encoded_comments = this.callback.args[0][4] + const encoded_comment = encoded_comments.comments.pop() + const encoded_comment_text = encoded_comment.op.c + return encoded_comment_text.should.equal('räksmörgÃ¥s') + }) + }) + + describe('with changes that need encoding', function () { + it('should call the callback with the encoded insert change', function () { + this.ranges.changes = [{ op: { i: 'räksmörgås' } }] + this.WebsocketController.joinDoc( + this.client, + this.doc_id, + -1, + { encodeRanges: true }, + this.callback + ) + + const encoded_changes = this.callback.args[0][4] + const encoded_change = encoded_changes.changes.pop() + const encoded_change_text = encoded_change.op.i + return encoded_change_text.should.equal('räksmörgÃ¥s') + }) + + return it('should call the callback with the encoded delete change', function () { + this.ranges.changes = [{ op: { d: 'räksmörgås' } }] + this.WebsocketController.joinDoc( + this.client, + this.doc_id, + -1, + { encodeRanges: true }, + this.callback + ) + + const encoded_changes = this.callback.args[0][4] + const encoded_change = encoded_changes.changes.pop() + const encoded_change_text = encoded_change.op.d + return encoded_change_text.should.equal('räksmörgÃ¥s') + }) + }) + + describe('when not authorized', function () { + beforeEach(function () { + this.AuthorizationManager.assertClientCanViewProject = sinon + .stub() + .callsArgWith(1, (this.err = new Error('not authorized'))) + return this.WebsocketController.joinDoc( + this.client, + this.doc_id, + -1, + this.options, + this.callback + ) + }) + + it('should call the callback with an error', function () { + return this.callback + .calledWith(sinon.match({ message: 'not authorized' })) + .should.equal(true) + }) + + return it('should not call the DocumentUpdaterManager', function () { + return this.DocumentUpdaterManager.getDocument.called.should.equal( + false + ) + }) + }) + + describe('with a restricted client', function () { + beforeEach(function () { + this.ranges.comments = [{ op: { a: 1 } }, { op: { a: 2 } }] + this.client.ol_context.is_restricted_user = true + return this.WebsocketController.joinDoc( + this.client, + this.doc_id, + -1, + this.options, + this.callback + ) + }) + + return it('should overwrite ranges.comments with an empty list', function () { + const ranges = this.callback.args[0][4] + return expect(ranges.comments).to.deep.equal([]) + }) + }) + + describe('when the client has disconnected', function () { + beforeEach(function () { + this.client.disconnected = true + return this.WebsocketController.joinDoc( + this.client, + this.doc_id, + -1, + this.options, + this.callback + ) + }) + + it('should call the callback with no details', function () { + return expect(this.callback.args[0]).to.deep.equal([]) + }) + + it('should increment the editor.join-doc.disconnected metric with a status', function () { + return expect( + this.metrics.inc.calledWith('editor.join-doc.disconnected', 1, { + status: 'immediately' + }) + ).to.equal(true) + }) + + return it('should not get the document', function () { + return expect(this.DocumentUpdaterManager.getDocument.called).to.equal( + false + ) + }) + }) + + describe('when the client disconnects while RoomManager.joinDoc is running', function () { + beforeEach(function () { + this.RoomManager.joinDoc = (client, doc_id, cb) => { + this.client.disconnected = true + return cb() + } + + return this.WebsocketController.joinDoc( + this.client, + this.doc_id, + -1, + this.options, + this.callback + ) + }) + + it('should call the callback with no details', function () { + return expect(this.callback.args[0]).to.deep.equal([]) + }) + + it('should increment the editor.join-doc.disconnected metric with a status', function () { + return expect( + this.metrics.inc.calledWith('editor.join-doc.disconnected', 1, { + status: 'after-joining-room' + }) + ).to.equal(true) + }) + + return it('should not get the document', function () { + return expect(this.DocumentUpdaterManager.getDocument.called).to.equal( + false + ) + }) + }) + + return describe('when the client disconnects while DocumentUpdaterManager.getDocument is running', function () { + beforeEach(function () { + this.DocumentUpdaterManager.getDocument = ( + project_id, + doc_id, + fromVersion, + callback + ) => { + this.client.disconnected = true + return callback( + null, + this.doc_lines, + this.version, + this.ranges, + this.ops + ) + } + + return this.WebsocketController.joinDoc( + this.client, + this.doc_id, + -1, + this.options, + this.callback + ) + }) + + it('should call the callback with no details', function () { + return expect(this.callback.args[0]).to.deep.equal([]) + }) + + return it('should increment the editor.join-doc.disconnected metric with a status', function () { + return expect( + this.metrics.inc.calledWith('editor.join-doc.disconnected', 1, { + status: 'after-doc-updater-call' + }) + ).to.equal(true) + }) + }) + }) + + describe('leaveDoc', function () { + beforeEach(function () { + this.doc_id = 'doc-id-123' + this.client.ol_context.project_id = this.project_id + this.RoomManager.leaveDoc = sinon.stub() + return this.WebsocketController.leaveDoc( + this.client, + this.doc_id, + this.callback + ) + }) + + it('should remove the client from the doc_id room', function () { + return this.RoomManager.leaveDoc + .calledWith(this.client, this.doc_id) + .should.equal(true) + }) + + it('should call the callback', function () { + return this.callback.called.should.equal(true) + }) + + return it('should increment the leave-doc metric', function () { + return this.metrics.inc.calledWith('editor.leave-doc').should.equal(true) + }) + }) + + describe('getConnectedUsers', function () { + beforeEach(function () { + this.client.ol_context.project_id = this.project_id + this.users = ['mock', 'users'] + this.WebsocketLoadBalancer.emitToRoom = sinon.stub() + return (this.ConnectedUsersManager.getConnectedUsers = sinon + .stub() + .callsArgWith(1, null, this.users)) + }) + + describe('when authorized', function () { + beforeEach(function (done) { + this.AuthorizationManager.assertClientCanViewProject = sinon + .stub() + .callsArgWith(1, null) + return this.WebsocketController.getConnectedUsers( + this.client, + (...args) => { + this.callback(...Array.from(args || [])) + return done() + } + ) + }) + + it('should check that the client is authorized to view the project', function () { + return this.AuthorizationManager.assertClientCanViewProject + .calledWith(this.client) + .should.equal(true) + }) + + it('should broadcast a request to update the client list', function () { + return this.WebsocketLoadBalancer.emitToRoom + .calledWith(this.project_id, 'clientTracking.refresh') + .should.equal(true) + }) + + it('should get the connected users for the project', function () { + return this.ConnectedUsersManager.getConnectedUsers + .calledWith(this.project_id) + .should.equal(true) + }) + + it('should return the users', function () { + return this.callback.calledWith(null, this.users).should.equal(true) + }) + + return it('should increment the get-connected-users metric', function () { + return this.metrics.inc + .calledWith('editor.get-connected-users') + .should.equal(true) + }) + }) + + describe('when not authorized', function () { + beforeEach(function () { + this.AuthorizationManager.assertClientCanViewProject = sinon + .stub() + .callsArgWith(1, (this.err = new Error('not authorized'))) + return this.WebsocketController.getConnectedUsers( + this.client, + this.callback + ) + }) + + it('should not get the connected users for the project', function () { + return this.ConnectedUsersManager.getConnectedUsers.called.should.equal( + false + ) + }) + + return it('should return an error', function () { + return this.callback.calledWith(this.err).should.equal(true) + }) + }) + + describe('when restricted user', function () { + beforeEach(function () { + this.client.ol_context.is_restricted_user = true + this.AuthorizationManager.assertClientCanViewProject = sinon + .stub() + .callsArgWith(1, null) + return this.WebsocketController.getConnectedUsers( + this.client, + this.callback + ) + }) + + it('should return an empty array of users', function () { + return this.callback.calledWith(null, []).should.equal(true) + }) + + return it('should not get the connected users for the project', function () { + return this.ConnectedUsersManager.getConnectedUsers.called.should.equal( + false + ) + }) + }) + + return describe('when the client has disconnected', function () { + beforeEach(function () { + this.client.disconnected = true + this.AuthorizationManager.assertClientCanViewProject = sinon.stub() + return this.WebsocketController.getConnectedUsers( + this.client, + this.callback + ) + }) + + it('should call the callback with no details', function () { + return expect(this.callback.args[0]).to.deep.equal([]) + }) + + return it('should not check permissions', function () { + return expect( + this.AuthorizationManager.assertClientCanViewProject.called + ).to.equal(false) + }) + }) + }) + + describe('updateClientPosition', function () { + beforeEach(function () { + this.WebsocketLoadBalancer.emitToRoom = sinon.stub() + this.ConnectedUsersManager.updateUserPosition = sinon + .stub() + .callsArgWith(4) + this.AuthorizationManager.assertClientCanViewProjectAndDoc = sinon + .stub() + .callsArgWith(2, null) + return (this.update = { + doc_id: (this.doc_id = 'doc-id-123'), + row: (this.row = 42), + column: (this.column = 37) + }) + }) + + describe('with a logged in user', function () { + beforeEach(function () { + this.client.ol_context = { + project_id: this.project_id, + first_name: (this.first_name = 'Douglas'), + last_name: (this.last_name = 'Adams'), + email: (this.email = 'joe@example.com'), + user_id: (this.user_id = 'user-id-123') + } + this.WebsocketController.updateClientPosition(this.client, this.update) + + return (this.populatedCursorData = { + doc_id: this.doc_id, + id: this.client.publicId, + name: `${this.first_name} ${this.last_name}`, + row: this.row, + column: this.column, + email: this.email, + user_id: this.user_id + }) + }) + + it("should send the update to the project room with the user's name", function () { + return this.WebsocketLoadBalancer.emitToRoom + .calledWith( + this.project_id, + 'clientTracking.clientUpdated', + this.populatedCursorData + ) + .should.equal(true) + }) + + it('should send the cursor data to the connected user manager', function (done) { + this.ConnectedUsersManager.updateUserPosition + .calledWith( + this.project_id, + this.client.publicId, + { + _id: this.user_id, + email: this.email, + first_name: this.first_name, + last_name: this.last_name + }, + { + row: this.row, + column: this.column, + doc_id: this.doc_id + } + ) + .should.equal(true) + return done() + }) + + return it('should increment the update-client-position metric at 0.1 frequency', function () { + return this.metrics.inc + .calledWith('editor.update-client-position', 0.1) + .should.equal(true) + }) + }) + + describe('with a logged in user who has no last_name set', function () { + beforeEach(function () { + this.client.ol_context = { + project_id: this.project_id, + first_name: (this.first_name = 'Douglas'), + last_name: undefined, + email: (this.email = 'joe@example.com'), + user_id: (this.user_id = 'user-id-123') + } + this.WebsocketController.updateClientPosition(this.client, this.update) + + return (this.populatedCursorData = { + doc_id: this.doc_id, + id: this.client.publicId, + name: `${this.first_name}`, + row: this.row, + column: this.column, + email: this.email, + user_id: this.user_id + }) + }) + + it("should send the update to the project room with the user's name", function () { + return this.WebsocketLoadBalancer.emitToRoom + .calledWith( + this.project_id, + 'clientTracking.clientUpdated', + this.populatedCursorData + ) + .should.equal(true) + }) + + it('should send the cursor data to the connected user manager', function (done) { + this.ConnectedUsersManager.updateUserPosition + .calledWith( + this.project_id, + this.client.publicId, + { + _id: this.user_id, + email: this.email, + first_name: this.first_name, + last_name: undefined + }, + { + row: this.row, + column: this.column, + doc_id: this.doc_id + } + ) + .should.equal(true) + return done() + }) + + return it('should increment the update-client-position metric at 0.1 frequency', function () { + return this.metrics.inc + .calledWith('editor.update-client-position', 0.1) + .should.equal(true) + }) + }) + + describe('with a logged in user who has no first_name set', function () { + beforeEach(function () { + this.client.ol_context = { + project_id: this.project_id, + first_name: undefined, + last_name: (this.last_name = 'Adams'), + email: (this.email = 'joe@example.com'), + user_id: (this.user_id = 'user-id-123') + } + this.WebsocketController.updateClientPosition(this.client, this.update) + + return (this.populatedCursorData = { + doc_id: this.doc_id, + id: this.client.publicId, + name: `${this.last_name}`, + row: this.row, + column: this.column, + email: this.email, + user_id: this.user_id + }) + }) + + it("should send the update to the project room with the user's name", function () { + return this.WebsocketLoadBalancer.emitToRoom + .calledWith( + this.project_id, + 'clientTracking.clientUpdated', + this.populatedCursorData + ) + .should.equal(true) + }) + + it('should send the cursor data to the connected user manager', function (done) { + this.ConnectedUsersManager.updateUserPosition + .calledWith( + this.project_id, + this.client.publicId, + { + _id: this.user_id, + email: this.email, + first_name: undefined, + last_name: this.last_name + }, + { + row: this.row, + column: this.column, + doc_id: this.doc_id + } + ) + .should.equal(true) + return done() + }) + + return it('should increment the update-client-position metric at 0.1 frequency', function () { + return this.metrics.inc + .calledWith('editor.update-client-position', 0.1) + .should.equal(true) + }) + }) + describe('with a logged in user who has no names set', function () { + beforeEach(function () { + this.client.ol_context = { + project_id: this.project_id, + first_name: undefined, + last_name: undefined, + email: (this.email = 'joe@example.com'), + user_id: (this.user_id = 'user-id-123') + } + return this.WebsocketController.updateClientPosition( + this.client, + this.update + ) + }) + + return it('should send the update to the project name with no name', function () { + return this.WebsocketLoadBalancer.emitToRoom + .calledWith(this.project_id, 'clientTracking.clientUpdated', { + doc_id: this.doc_id, + id: this.client.publicId, + user_id: this.user_id, + name: '', + row: this.row, + column: this.column, + email: this.email + }) + .should.equal(true) + }) + }) + + describe('with an anonymous user', function () { + beforeEach(function () { + this.client.ol_context = { + project_id: this.project_id + } + return this.WebsocketController.updateClientPosition( + this.client, + this.update + ) + }) + + it('should send the update to the project room with no name', function () { + return this.WebsocketLoadBalancer.emitToRoom + .calledWith(this.project_id, 'clientTracking.clientUpdated', { + doc_id: this.doc_id, + id: this.client.publicId, + name: '', + row: this.row, + column: this.column + }) + .should.equal(true) + }) + + return it('should not send cursor data to the connected user manager', function (done) { + this.ConnectedUsersManager.updateUserPosition.called.should.equal(false) + return done() + }) + }) + + return describe('when the client has disconnected', function () { + beforeEach(function () { + this.client.disconnected = true + this.AuthorizationManager.assertClientCanViewProjectAndDoc = sinon.stub() + return this.WebsocketController.updateClientPosition( + this.client, + this.update, + this.callback + ) + }) + + it('should call the callback with no details', function () { + return expect(this.callback.args[0]).to.deep.equal([]) + }) + + return it('should not check permissions', function () { + return expect( + this.AuthorizationManager.assertClientCanViewProjectAndDoc.called + ).to.equal(false) + }) + }) + }) + + describe('applyOtUpdate', function () { + beforeEach(function () { + this.update = { op: { p: 12, t: 'foo' } } + this.client.ol_context.user_id = this.user_id + this.client.ol_context.project_id = this.project_id + this.WebsocketController._assertClientCanApplyUpdate = sinon + .stub() + .yields() + return (this.DocumentUpdaterManager.queueChange = sinon + .stub() + .callsArg(3)) + }) + + describe('succesfully', function () { + beforeEach(function () { + return this.WebsocketController.applyOtUpdate( + this.client, + this.doc_id, + this.update, + this.callback + ) + }) + + it('should set the source of the update to the client id', function () { + return this.update.meta.source.should.equal(this.client.publicId) + }) + + it('should set the user_id of the update to the user id', function () { + return this.update.meta.user_id.should.equal(this.user_id) + }) + + it('should queue the update', function () { + return this.DocumentUpdaterManager.queueChange + .calledWith(this.project_id, this.doc_id, this.update) + .should.equal(true) + }) + + it('should call the callback', function () { + return this.callback.called.should.equal(true) + }) + + return it('should increment the doc updates', function () { + return this.metrics.inc + .calledWith('editor.doc-update') + .should.equal(true) + }) + }) + + describe('unsuccessfully', function () { + beforeEach(function () { + this.client.disconnect = sinon.stub() + this.DocumentUpdaterManager.queueChange = sinon + .stub() + .callsArgWith(3, (this.error = new Error('Something went wrong'))) + return this.WebsocketController.applyOtUpdate( + this.client, + this.doc_id, + this.update, + this.callback + ) + }) + + it('should disconnect the client', function () { + return this.client.disconnect.called.should.equal(true) + }) + + it('should log an error', function () { + return this.logger.error.called.should.equal(true) + }) + + return it('should call the callback with the error', function () { + return this.callback.calledWith(this.error).should.equal(true) + }) + }) + + describe('when not authorized', function () { + beforeEach(function () { + this.client.disconnect = sinon.stub() + this.WebsocketController._assertClientCanApplyUpdate = sinon + .stub() + .yields((this.error = new Error('not authorized'))) + return this.WebsocketController.applyOtUpdate( + this.client, + this.doc_id, + this.update, + this.callback + ) + }) + + // This happens in a setTimeout to allow the client a chance to receive the error first. + // I'm not sure how to unit test, but it is acceptance tested. + // it "should disconnect the client", -> + // @client.disconnect.called.should.equal true + + it('should log a warning', function () { + return this.logger.warn.called.should.equal(true) + }) + + return it('should call the callback with the error', function () { + return this.callback.calledWith(this.error).should.equal(true) + }) + }) + + return describe('update_too_large', function () { + beforeEach(function (done) { + this.client.disconnect = sinon.stub() + this.client.emit = sinon.stub() + this.client.ol_context.user_id = this.user_id + this.client.ol_context.project_id = this.project_id + const error = new Error('update is too large') + error.updateSize = 7372835 + this.DocumentUpdaterManager.queueChange = sinon + .stub() + .callsArgWith(3, error) + this.WebsocketController.applyOtUpdate( + this.client, + this.doc_id, + this.update, + this.callback + ) + return setTimeout(() => done(), 1) + }) + + it('should call the callback with no error', function () { + this.callback.called.should.equal(true) + return this.callback.args[0].should.deep.equal([]) + }) + + it('should log a warning with the size and context', function () { + this.logger.warn.called.should.equal(true) + return this.logger.warn.args[0].should.deep.equal([ + { + user_id: this.user_id, + project_id: this.project_id, + doc_id: this.doc_id, + updateSize: 7372835 + }, + 'update is too large' + ]) + }) + + describe('after 100ms', function () { + beforeEach(function (done) { + return setTimeout(done, 100) + }) + + it('should send an otUpdateError the client', function () { + return this.client.emit.calledWith('otUpdateError').should.equal(true) + }) + + return it('should disconnect the client', function () { + return this.client.disconnect.called.should.equal(true) + }) + }) + + return describe('when the client disconnects during the next 100ms', function () { + beforeEach(function (done) { + this.client.disconnected = true + return setTimeout(done, 100) + }) + + it('should not send an otUpdateError the client', function () { + return this.client.emit + .calledWith('otUpdateError') + .should.equal(false) + }) + + it('should not disconnect the client', function () { + return this.client.disconnect.called.should.equal(false) + }) + + return it('should increment the editor.doc-update.disconnected metric with a status', function () { + return expect( + this.metrics.inc.calledWith('editor.doc-update.disconnected', 1, { + status: 'at-otUpdateError' + }) + ).to.equal(true) + }) + }) + }) + }) + + return describe('_assertClientCanApplyUpdate', function () { + beforeEach(function () { + this.edit_update = { + op: [ + { i: 'foo', p: 42 }, + { c: 'bar', p: 132 } + ] + } // comments may still be in an edit op + this.comment_update = { op: [{ c: 'bar', p: 132 }] } + this.AuthorizationManager.assertClientCanEditProjectAndDoc = sinon.stub() + return (this.AuthorizationManager.assertClientCanViewProjectAndDoc = sinon.stub()) + }) + + describe('with a read-write client', function () { + return it('should return successfully', function (done) { + this.AuthorizationManager.assertClientCanEditProjectAndDoc.yields(null) + return this.WebsocketController._assertClientCanApplyUpdate( + this.client, + this.doc_id, + this.edit_update, + (error) => { + expect(error).to.be.null + return done() + } + ) + }) + }) + + describe('with a read-only client and an edit op', function () { + return it('should return an error', function (done) { + this.AuthorizationManager.assertClientCanEditProjectAndDoc.yields( + new Error('not authorized') + ) + this.AuthorizationManager.assertClientCanViewProjectAndDoc.yields(null) + return this.WebsocketController._assertClientCanApplyUpdate( + this.client, + this.doc_id, + this.edit_update, + (error) => { + expect(error.message).to.equal('not authorized') + return done() + } + ) + }) + }) + + describe('with a read-only client and a comment op', function () { + return it('should return successfully', function (done) { + this.AuthorizationManager.assertClientCanEditProjectAndDoc.yields( + new Error('not authorized') + ) + this.AuthorizationManager.assertClientCanViewProjectAndDoc.yields(null) + return this.WebsocketController._assertClientCanApplyUpdate( + this.client, + this.doc_id, + this.comment_update, + (error) => { + expect(error).to.be.null + return done() + } + ) + }) + }) + + return describe('with a totally unauthorized client', function () { + return it('should return an error', function (done) { + this.AuthorizationManager.assertClientCanEditProjectAndDoc.yields( + new Error('not authorized') + ) + this.AuthorizationManager.assertClientCanViewProjectAndDoc.yields( + new Error('not authorized') + ) + return this.WebsocketController._assertClientCanApplyUpdate( + this.client, + this.doc_id, + this.comment_update, + (error) => { + expect(error.message).to.equal('not authorized') + return done() + } + ) + }) + }) + }) +}) diff --git a/services/real-time/test/unit/js/WebsocketLoadBalancerTests.js b/services/real-time/test/unit/js/WebsocketLoadBalancerTests.js index 0d0c0f6b9d..355e635353 100644 --- a/services/real-time/test/unit/js/WebsocketLoadBalancerTests.js +++ b/services/real-time/test/unit/js/WebsocketLoadBalancerTests.js @@ -9,201 +9,301 @@ * DS102: Remove unnecessary code created because of implicit returns * Full docs: https://github.com/decaffeinate/decaffeinate/blob/master/docs/suggestions.md */ -const SandboxedModule = require('sandboxed-module'); -const sinon = require('sinon'); -require('chai').should(); -const modulePath = require('path').join(__dirname, '../../../app/js/WebsocketLoadBalancer'); +const SandboxedModule = require('sandboxed-module') +const sinon = require('sinon') +require('chai').should() +const modulePath = require('path').join( + __dirname, + '../../../app/js/WebsocketLoadBalancer' +) -describe("WebsocketLoadBalancer", function() { - beforeEach(function() { - this.rclient = {}; - this.RoomEvents = {on: sinon.stub()}; - this.WebsocketLoadBalancer = SandboxedModule.require(modulePath, { requires: { - "./RedisClientManager": { - createClientList: () => [] - }, - "logger-sharelatex": (this.logger = { log: sinon.stub(), error: sinon.stub() }), - "./SafeJsonParse": (this.SafeJsonParse = - {parse: (data, cb) => cb(null, JSON.parse(data))}), - "./EventLogger": {checkEventOrder: sinon.stub()}, - "./HealthCheckManager": {check: sinon.stub()}, - "./RoomManager" : (this.RoomManager = {eventSource: sinon.stub().returns(this.RoomEvents)}), - "./ChannelManager": (this.ChannelManager = {publish: sinon.stub()}), - "./ConnectedUsersManager": (this.ConnectedUsersManager = {refreshClient: sinon.stub()}) - } - }); - this.io = {}; - this.WebsocketLoadBalancer.rclientPubList = [{publish: sinon.stub()}]; - this.WebsocketLoadBalancer.rclientSubList = [{ - subscribe: sinon.stub(), - on: sinon.stub() - }]; +describe('WebsocketLoadBalancer', function () { + beforeEach(function () { + this.rclient = {} + this.RoomEvents = { on: sinon.stub() } + this.WebsocketLoadBalancer = SandboxedModule.require(modulePath, { + requires: { + './RedisClientManager': { + createClientList: () => [] + }, + 'logger-sharelatex': (this.logger = { + log: sinon.stub(), + error: sinon.stub() + }), + './SafeJsonParse': (this.SafeJsonParse = { + parse: (data, cb) => cb(null, JSON.parse(data)) + }), + './EventLogger': { checkEventOrder: sinon.stub() }, + './HealthCheckManager': { check: sinon.stub() }, + './RoomManager': (this.RoomManager = { + eventSource: sinon.stub().returns(this.RoomEvents) + }), + './ChannelManager': (this.ChannelManager = { publish: sinon.stub() }), + './ConnectedUsersManager': (this.ConnectedUsersManager = { + refreshClient: sinon.stub() + }) + } + }) + this.io = {} + this.WebsocketLoadBalancer.rclientPubList = [{ publish: sinon.stub() }] + this.WebsocketLoadBalancer.rclientSubList = [ + { + subscribe: sinon.stub(), + on: sinon.stub() + } + ] - this.room_id = "room-id"; - this.message = "otUpdateApplied"; - return this.payload = ["argument one", 42];}); + this.room_id = 'room-id' + this.message = 'otUpdateApplied' + return (this.payload = ['argument one', 42]) + }) - describe("emitToRoom", function() { - beforeEach(function() { - return this.WebsocketLoadBalancer.emitToRoom(this.room_id, this.message, ...Array.from(this.payload)); - }); + describe('emitToRoom', function () { + beforeEach(function () { + return this.WebsocketLoadBalancer.emitToRoom( + this.room_id, + this.message, + ...Array.from(this.payload) + ) + }) - return it("should publish the message to redis", function() { - return this.ChannelManager.publish - .calledWith(this.WebsocketLoadBalancer.rclientPubList[0], "editor-events", this.room_id, JSON.stringify({ - room_id: this.room_id, - message: this.message, - payload: this.payload - })) - .should.equal(true); - }); - }); + return it('should publish the message to redis', function () { + return this.ChannelManager.publish + .calledWith( + this.WebsocketLoadBalancer.rclientPubList[0], + 'editor-events', + this.room_id, + JSON.stringify({ + room_id: this.room_id, + message: this.message, + payload: this.payload + }) + ) + .should.equal(true) + }) + }) - describe("emitToAll", function() { - beforeEach(function() { - this.WebsocketLoadBalancer.emitToRoom = sinon.stub(); - return this.WebsocketLoadBalancer.emitToAll(this.message, ...Array.from(this.payload)); - }); + describe('emitToAll', function () { + beforeEach(function () { + this.WebsocketLoadBalancer.emitToRoom = sinon.stub() + return this.WebsocketLoadBalancer.emitToAll( + this.message, + ...Array.from(this.payload) + ) + }) - return it("should emit to the room 'all'", function() { - return this.WebsocketLoadBalancer.emitToRoom - .calledWith("all", this.message, ...Array.from(this.payload)) - .should.equal(true); - }); - }); + return it("should emit to the room 'all'", function () { + return this.WebsocketLoadBalancer.emitToRoom + .calledWith('all', this.message, ...Array.from(this.payload)) + .should.equal(true) + }) + }) - describe("listenForEditorEvents", function() { - beforeEach(function() { - this.WebsocketLoadBalancer._processEditorEvent = sinon.stub(); - return this.WebsocketLoadBalancer.listenForEditorEvents(); - }); + describe('listenForEditorEvents', function () { + beforeEach(function () { + this.WebsocketLoadBalancer._processEditorEvent = sinon.stub() + return this.WebsocketLoadBalancer.listenForEditorEvents() + }) - it("should subscribe to the editor-events channel", function() { - return this.WebsocketLoadBalancer.rclientSubList[0].subscribe - .calledWith("editor-events") - .should.equal(true); - }); + it('should subscribe to the editor-events channel', function () { + return this.WebsocketLoadBalancer.rclientSubList[0].subscribe + .calledWith('editor-events') + .should.equal(true) + }) - return it("should process the events with _processEditorEvent", function() { - return this.WebsocketLoadBalancer.rclientSubList[0].on - .calledWith("message", sinon.match.func) - .should.equal(true); - }); - }); + return it('should process the events with _processEditorEvent', function () { + return this.WebsocketLoadBalancer.rclientSubList[0].on + .calledWith('message', sinon.match.func) + .should.equal(true) + }) + }) - return describe("_processEditorEvent", function() { - describe("with bad JSON", function() { - beforeEach(function() { - this.isRestrictedUser = false; - this.SafeJsonParse.parse = sinon.stub().callsArgWith(1, new Error("oops")); - return this.WebsocketLoadBalancer._processEditorEvent(this.io, "editor-events", "blah"); - }); + return describe('_processEditorEvent', function () { + describe('with bad JSON', function () { + beforeEach(function () { + this.isRestrictedUser = false + this.SafeJsonParse.parse = sinon + .stub() + .callsArgWith(1, new Error('oops')) + return this.WebsocketLoadBalancer._processEditorEvent( + this.io, + 'editor-events', + 'blah' + ) + }) - return it("should log an error", function() { - return this.logger.error.called.should.equal(true); - }); - }); + return it('should log an error', function () { + return this.logger.error.called.should.equal(true) + }) + }) - describe("with a designated room", function() { - beforeEach(function() { - this.io.sockets = { - clients: sinon.stub().returns([ - {id: 'client-id-1', emit: (this.emit1 = sinon.stub()), ol_context: {}}, - {id: 'client-id-2', emit: (this.emit2 = sinon.stub()), ol_context: {}}, - {id: 'client-id-1', emit: (this.emit3 = sinon.stub()), ol_context: {}} // duplicate client - ]) - }; - const data = JSON.stringify({ - room_id: this.room_id, - message: this.message, - payload: this.payload - }); - return this.WebsocketLoadBalancer._processEditorEvent(this.io, "editor-events", data); - }); + describe('with a designated room', function () { + beforeEach(function () { + this.io.sockets = { + clients: sinon.stub().returns([ + { + id: 'client-id-1', + emit: (this.emit1 = sinon.stub()), + ol_context: {} + }, + { + id: 'client-id-2', + emit: (this.emit2 = sinon.stub()), + ol_context: {} + }, + { + id: 'client-id-1', + emit: (this.emit3 = sinon.stub()), + ol_context: {} + } // duplicate client + ]) + } + const data = JSON.stringify({ + room_id: this.room_id, + message: this.message, + payload: this.payload + }) + return this.WebsocketLoadBalancer._processEditorEvent( + this.io, + 'editor-events', + data + ) + }) - return it("should send the message to all (unique) clients in the room", function() { - this.io.sockets.clients - .calledWith(this.room_id) - .should.equal(true); - this.emit1.calledWith(this.message, ...Array.from(this.payload)).should.equal(true); - this.emit2.calledWith(this.message, ...Array.from(this.payload)).should.equal(true); - return this.emit3.called.should.equal(false); - }); - }); // duplicate client should be ignored + return it('should send the message to all (unique) clients in the room', function () { + this.io.sockets.clients.calledWith(this.room_id).should.equal(true) + this.emit1 + .calledWith(this.message, ...Array.from(this.payload)) + .should.equal(true) + this.emit2 + .calledWith(this.message, ...Array.from(this.payload)) + .should.equal(true) + return this.emit3.called.should.equal(false) + }) + }) // duplicate client should be ignored - describe("with a designated room, and restricted clients, not restricted message", function() { - beforeEach(function() { - this.io.sockets = { - clients: sinon.stub().returns([ - {id: 'client-id-1', emit: (this.emit1 = sinon.stub()), ol_context: {}}, - {id: 'client-id-2', emit: (this.emit2 = sinon.stub()), ol_context: {}}, - {id: 'client-id-1', emit: (this.emit3 = sinon.stub()), ol_context: {}}, // duplicate client - {id: 'client-id-4', emit: (this.emit4 = sinon.stub()), ol_context: {is_restricted_user: true}} - ]) - }; - const data = JSON.stringify({ - room_id: this.room_id, - message: this.message, - payload: this.payload - }); - return this.WebsocketLoadBalancer._processEditorEvent(this.io, "editor-events", data); - }); + describe('with a designated room, and restricted clients, not restricted message', function () { + beforeEach(function () { + this.io.sockets = { + clients: sinon.stub().returns([ + { + id: 'client-id-1', + emit: (this.emit1 = sinon.stub()), + ol_context: {} + }, + { + id: 'client-id-2', + emit: (this.emit2 = sinon.stub()), + ol_context: {} + }, + { + id: 'client-id-1', + emit: (this.emit3 = sinon.stub()), + ol_context: {} + }, // duplicate client + { + id: 'client-id-4', + emit: (this.emit4 = sinon.stub()), + ol_context: { is_restricted_user: true } + } + ]) + } + const data = JSON.stringify({ + room_id: this.room_id, + message: this.message, + payload: this.payload + }) + return this.WebsocketLoadBalancer._processEditorEvent( + this.io, + 'editor-events', + data + ) + }) - return it("should send the message to all (unique) clients in the room", function() { - this.io.sockets.clients - .calledWith(this.room_id) - .should.equal(true); - this.emit1.calledWith(this.message, ...Array.from(this.payload)).should.equal(true); - this.emit2.calledWith(this.message, ...Array.from(this.payload)).should.equal(true); - this.emit3.called.should.equal(false); // duplicate client should be ignored - return this.emit4.called.should.equal(true); - }); - }); // restricted client, but should be called + return it('should send the message to all (unique) clients in the room', function () { + this.io.sockets.clients.calledWith(this.room_id).should.equal(true) + this.emit1 + .calledWith(this.message, ...Array.from(this.payload)) + .should.equal(true) + this.emit2 + .calledWith(this.message, ...Array.from(this.payload)) + .should.equal(true) + this.emit3.called.should.equal(false) // duplicate client should be ignored + return this.emit4.called.should.equal(true) + }) + }) // restricted client, but should be called - describe("with a designated room, and restricted clients, restricted message", function() { - beforeEach(function() { - this.io.sockets = { - clients: sinon.stub().returns([ - {id: 'client-id-1', emit: (this.emit1 = sinon.stub()), ol_context: {}}, - {id: 'client-id-2', emit: (this.emit2 = sinon.stub()), ol_context: {}}, - {id: 'client-id-1', emit: (this.emit3 = sinon.stub()), ol_context: {}}, // duplicate client - {id: 'client-id-4', emit: (this.emit4 = sinon.stub()), ol_context: {is_restricted_user: true}} - ]) - }; - const data = JSON.stringify({ - room_id: this.room_id, - message: (this.restrictedMessage = 'new-comment'), - payload: this.payload - }); - return this.WebsocketLoadBalancer._processEditorEvent(this.io, "editor-events", data); - }); + describe('with a designated room, and restricted clients, restricted message', function () { + beforeEach(function () { + this.io.sockets = { + clients: sinon.stub().returns([ + { + id: 'client-id-1', + emit: (this.emit1 = sinon.stub()), + ol_context: {} + }, + { + id: 'client-id-2', + emit: (this.emit2 = sinon.stub()), + ol_context: {} + }, + { + id: 'client-id-1', + emit: (this.emit3 = sinon.stub()), + ol_context: {} + }, // duplicate client + { + id: 'client-id-4', + emit: (this.emit4 = sinon.stub()), + ol_context: { is_restricted_user: true } + } + ]) + } + const data = JSON.stringify({ + room_id: this.room_id, + message: (this.restrictedMessage = 'new-comment'), + payload: this.payload + }) + return this.WebsocketLoadBalancer._processEditorEvent( + this.io, + 'editor-events', + data + ) + }) - return it("should send the message to all (unique) clients in the room, who are not restricted", function() { - this.io.sockets.clients - .calledWith(this.room_id) - .should.equal(true); - this.emit1.calledWith(this.restrictedMessage, ...Array.from(this.payload)).should.equal(true); - this.emit2.calledWith(this.restrictedMessage, ...Array.from(this.payload)).should.equal(true); - this.emit3.called.should.equal(false); // duplicate client should be ignored - return this.emit4.called.should.equal(false); - }); - }); // restricted client, should not be called + return it('should send the message to all (unique) clients in the room, who are not restricted', function () { + this.io.sockets.clients.calledWith(this.room_id).should.equal(true) + this.emit1 + .calledWith(this.restrictedMessage, ...Array.from(this.payload)) + .should.equal(true) + this.emit2 + .calledWith(this.restrictedMessage, ...Array.from(this.payload)) + .should.equal(true) + this.emit3.called.should.equal(false) // duplicate client should be ignored + return this.emit4.called.should.equal(false) + }) + }) // restricted client, should not be called - return describe("when emitting to all", function() { - beforeEach(function() { - this.io.sockets = - {emit: (this.emit = sinon.stub())}; - const data = JSON.stringify({ - room_id: "all", - message: this.message, - payload: this.payload - }); - return this.WebsocketLoadBalancer._processEditorEvent(this.io, "editor-events", data); - }); + return describe('when emitting to all', function () { + beforeEach(function () { + this.io.sockets = { emit: (this.emit = sinon.stub()) } + const data = JSON.stringify({ + room_id: 'all', + message: this.message, + payload: this.payload + }) + return this.WebsocketLoadBalancer._processEditorEvent( + this.io, + 'editor-events', + data + ) + }) - return it("should send the message to all clients", function() { - return this.emit.calledWith(this.message, ...Array.from(this.payload)).should.equal(true); - }); - }); - }); -}); + return it('should send the message to all clients', function () { + return this.emit + .calledWith(this.message, ...Array.from(this.payload)) + .should.equal(true) + }) + }) + }) +}) diff --git a/services/real-time/test/unit/js/helpers/MockClient.js b/services/real-time/test/unit/js/helpers/MockClient.js index 5f9b019db4..fb90a0f7e9 100644 --- a/services/real-time/test/unit/js/helpers/MockClient.js +++ b/services/real-time/test/unit/js/helpers/MockClient.js @@ -3,20 +3,20 @@ */ // TODO: This file was created by bulk-decaffeinate. // Fix any style issues and re-enable lint. -let MockClient; -const sinon = require('sinon'); +let MockClient +const sinon = require('sinon') -let idCounter = 0; +let idCounter = 0 -module.exports = (MockClient = class MockClient { - constructor() { - this.ol_context = {}; - this.join = sinon.stub(); - this.emit = sinon.stub(); - this.disconnect = sinon.stub(); - this.id = idCounter++; - this.publicId = idCounter++; - } +module.exports = MockClient = class MockClient { + constructor() { + this.ol_context = {} + this.join = sinon.stub() + this.emit = sinon.stub() + this.disconnect = sinon.stub() + this.id = idCounter++ + this.publicId = idCounter++ + } - disconnect() {} -}); + disconnect() {} +} From 30a9c6ed2c56da2d09f91052b78f3ff9e5426c1e Mon Sep 17 00:00:00 2001 From: decaffeinate Date: Tue, 23 Jun 2020 18:30:23 +0100 Subject: [PATCH 378/491] decaffeinate: Rename ApplyUpdateTests.coffee and 18 other files from .coffee to .js --- .../coffee/{ApplyUpdateTests.coffee => ApplyUpdateTests.js} | 0 .../coffee/{ClientTrackingTests.coffee => ClientTrackingTests.js} | 0 .../coffee/{DrainManagerTests.coffee => DrainManagerTests.js} | 0 .../coffee/{EarlyDisconnect.coffee => EarlyDisconnect.js} | 0 .../coffee/{HttpControllerTests.coffee => HttpControllerTests.js} | 0 .../acceptance/coffee/{JoinDocTests.coffee => JoinDocTests.js} | 0 .../coffee/{JoinProjectTests.coffee => JoinProjectTests.js} | 0 .../acceptance/coffee/{LeaveDocTests.coffee => LeaveDocTests.js} | 0 .../coffee/{LeaveProjectTests.coffee => LeaveProjectTests.js} | 0 .../test/acceptance/coffee/{PubSubRace.coffee => PubSubRace.js} | 0 .../coffee/{ReceiveUpdateTests.coffee => ReceiveUpdateTests.js} | 0 .../test/acceptance/coffee/{RouterTests.coffee => RouterTests.js} | 0 .../coffee/{SessionSocketsTests.coffee => SessionSocketsTests.js} | 0 .../acceptance/coffee/{SessionTests.coffee => SessionTests.js} | 0 .../coffee/helpers/{FixturesManager.coffee => FixturesManager.js} | 0 .../{MockDocUpdaterServer.coffee => MockDocUpdaterServer.js} | 0 .../coffee/helpers/{MockWebServer.coffee => MockWebServer.js} | 0 .../coffee/helpers/{RealTimeClient.coffee => RealTimeClient.js} | 0 .../coffee/helpers/{RealtimeServer.coffee => RealtimeServer.js} | 0 19 files changed, 0 insertions(+), 0 deletions(-) rename services/real-time/test/acceptance/coffee/{ApplyUpdateTests.coffee => ApplyUpdateTests.js} (100%) rename services/real-time/test/acceptance/coffee/{ClientTrackingTests.coffee => ClientTrackingTests.js} (100%) rename services/real-time/test/acceptance/coffee/{DrainManagerTests.coffee => DrainManagerTests.js} (100%) rename services/real-time/test/acceptance/coffee/{EarlyDisconnect.coffee => EarlyDisconnect.js} (100%) rename services/real-time/test/acceptance/coffee/{HttpControllerTests.coffee => HttpControllerTests.js} (100%) rename services/real-time/test/acceptance/coffee/{JoinDocTests.coffee => JoinDocTests.js} (100%) rename services/real-time/test/acceptance/coffee/{JoinProjectTests.coffee => JoinProjectTests.js} (100%) rename services/real-time/test/acceptance/coffee/{LeaveDocTests.coffee => LeaveDocTests.js} (100%) rename services/real-time/test/acceptance/coffee/{LeaveProjectTests.coffee => LeaveProjectTests.js} (100%) rename services/real-time/test/acceptance/coffee/{PubSubRace.coffee => PubSubRace.js} (100%) rename services/real-time/test/acceptance/coffee/{ReceiveUpdateTests.coffee => ReceiveUpdateTests.js} (100%) rename services/real-time/test/acceptance/coffee/{RouterTests.coffee => RouterTests.js} (100%) rename services/real-time/test/acceptance/coffee/{SessionSocketsTests.coffee => SessionSocketsTests.js} (100%) rename services/real-time/test/acceptance/coffee/{SessionTests.coffee => SessionTests.js} (100%) rename services/real-time/test/acceptance/coffee/helpers/{FixturesManager.coffee => FixturesManager.js} (100%) rename services/real-time/test/acceptance/coffee/helpers/{MockDocUpdaterServer.coffee => MockDocUpdaterServer.js} (100%) rename services/real-time/test/acceptance/coffee/helpers/{MockWebServer.coffee => MockWebServer.js} (100%) rename services/real-time/test/acceptance/coffee/helpers/{RealTimeClient.coffee => RealTimeClient.js} (100%) rename services/real-time/test/acceptance/coffee/helpers/{RealtimeServer.coffee => RealtimeServer.js} (100%) diff --git a/services/real-time/test/acceptance/coffee/ApplyUpdateTests.coffee b/services/real-time/test/acceptance/coffee/ApplyUpdateTests.js similarity index 100% rename from services/real-time/test/acceptance/coffee/ApplyUpdateTests.coffee rename to services/real-time/test/acceptance/coffee/ApplyUpdateTests.js diff --git a/services/real-time/test/acceptance/coffee/ClientTrackingTests.coffee b/services/real-time/test/acceptance/coffee/ClientTrackingTests.js similarity index 100% rename from services/real-time/test/acceptance/coffee/ClientTrackingTests.coffee rename to services/real-time/test/acceptance/coffee/ClientTrackingTests.js diff --git a/services/real-time/test/acceptance/coffee/DrainManagerTests.coffee b/services/real-time/test/acceptance/coffee/DrainManagerTests.js similarity index 100% rename from services/real-time/test/acceptance/coffee/DrainManagerTests.coffee rename to services/real-time/test/acceptance/coffee/DrainManagerTests.js diff --git a/services/real-time/test/acceptance/coffee/EarlyDisconnect.coffee b/services/real-time/test/acceptance/coffee/EarlyDisconnect.js similarity index 100% rename from services/real-time/test/acceptance/coffee/EarlyDisconnect.coffee rename to services/real-time/test/acceptance/coffee/EarlyDisconnect.js diff --git a/services/real-time/test/acceptance/coffee/HttpControllerTests.coffee b/services/real-time/test/acceptance/coffee/HttpControllerTests.js similarity index 100% rename from services/real-time/test/acceptance/coffee/HttpControllerTests.coffee rename to services/real-time/test/acceptance/coffee/HttpControllerTests.js diff --git a/services/real-time/test/acceptance/coffee/JoinDocTests.coffee b/services/real-time/test/acceptance/coffee/JoinDocTests.js similarity index 100% rename from services/real-time/test/acceptance/coffee/JoinDocTests.coffee rename to services/real-time/test/acceptance/coffee/JoinDocTests.js diff --git a/services/real-time/test/acceptance/coffee/JoinProjectTests.coffee b/services/real-time/test/acceptance/coffee/JoinProjectTests.js similarity index 100% rename from services/real-time/test/acceptance/coffee/JoinProjectTests.coffee rename to services/real-time/test/acceptance/coffee/JoinProjectTests.js diff --git a/services/real-time/test/acceptance/coffee/LeaveDocTests.coffee b/services/real-time/test/acceptance/coffee/LeaveDocTests.js similarity index 100% rename from services/real-time/test/acceptance/coffee/LeaveDocTests.coffee rename to services/real-time/test/acceptance/coffee/LeaveDocTests.js diff --git a/services/real-time/test/acceptance/coffee/LeaveProjectTests.coffee b/services/real-time/test/acceptance/coffee/LeaveProjectTests.js similarity index 100% rename from services/real-time/test/acceptance/coffee/LeaveProjectTests.coffee rename to services/real-time/test/acceptance/coffee/LeaveProjectTests.js diff --git a/services/real-time/test/acceptance/coffee/PubSubRace.coffee b/services/real-time/test/acceptance/coffee/PubSubRace.js similarity index 100% rename from services/real-time/test/acceptance/coffee/PubSubRace.coffee rename to services/real-time/test/acceptance/coffee/PubSubRace.js diff --git a/services/real-time/test/acceptance/coffee/ReceiveUpdateTests.coffee b/services/real-time/test/acceptance/coffee/ReceiveUpdateTests.js similarity index 100% rename from services/real-time/test/acceptance/coffee/ReceiveUpdateTests.coffee rename to services/real-time/test/acceptance/coffee/ReceiveUpdateTests.js diff --git a/services/real-time/test/acceptance/coffee/RouterTests.coffee b/services/real-time/test/acceptance/coffee/RouterTests.js similarity index 100% rename from services/real-time/test/acceptance/coffee/RouterTests.coffee rename to services/real-time/test/acceptance/coffee/RouterTests.js diff --git a/services/real-time/test/acceptance/coffee/SessionSocketsTests.coffee b/services/real-time/test/acceptance/coffee/SessionSocketsTests.js similarity index 100% rename from services/real-time/test/acceptance/coffee/SessionSocketsTests.coffee rename to services/real-time/test/acceptance/coffee/SessionSocketsTests.js diff --git a/services/real-time/test/acceptance/coffee/SessionTests.coffee b/services/real-time/test/acceptance/coffee/SessionTests.js similarity index 100% rename from services/real-time/test/acceptance/coffee/SessionTests.coffee rename to services/real-time/test/acceptance/coffee/SessionTests.js diff --git a/services/real-time/test/acceptance/coffee/helpers/FixturesManager.coffee b/services/real-time/test/acceptance/coffee/helpers/FixturesManager.js similarity index 100% rename from services/real-time/test/acceptance/coffee/helpers/FixturesManager.coffee rename to services/real-time/test/acceptance/coffee/helpers/FixturesManager.js diff --git a/services/real-time/test/acceptance/coffee/helpers/MockDocUpdaterServer.coffee b/services/real-time/test/acceptance/coffee/helpers/MockDocUpdaterServer.js similarity index 100% rename from services/real-time/test/acceptance/coffee/helpers/MockDocUpdaterServer.coffee rename to services/real-time/test/acceptance/coffee/helpers/MockDocUpdaterServer.js diff --git a/services/real-time/test/acceptance/coffee/helpers/MockWebServer.coffee b/services/real-time/test/acceptance/coffee/helpers/MockWebServer.js similarity index 100% rename from services/real-time/test/acceptance/coffee/helpers/MockWebServer.coffee rename to services/real-time/test/acceptance/coffee/helpers/MockWebServer.js diff --git a/services/real-time/test/acceptance/coffee/helpers/RealTimeClient.coffee b/services/real-time/test/acceptance/coffee/helpers/RealTimeClient.js similarity index 100% rename from services/real-time/test/acceptance/coffee/helpers/RealTimeClient.coffee rename to services/real-time/test/acceptance/coffee/helpers/RealTimeClient.js diff --git a/services/real-time/test/acceptance/coffee/helpers/RealtimeServer.coffee b/services/real-time/test/acceptance/coffee/helpers/RealtimeServer.js similarity index 100% rename from services/real-time/test/acceptance/coffee/helpers/RealtimeServer.coffee rename to services/real-time/test/acceptance/coffee/helpers/RealtimeServer.js From d318e4fd0edd3edbb83dc59e149696e67e7e90f8 Mon Sep 17 00:00:00 2001 From: decaffeinate Date: Tue, 23 Jun 2020 18:30:29 +0100 Subject: [PATCH 379/491] decaffeinate: Convert ApplyUpdateTests.coffee and 18 other files to JS --- .../acceptance/coffee/ApplyUpdateTests.js | 491 +++++++++++------- .../acceptance/coffee/ClientTrackingTests.js | 271 ++++++---- .../acceptance/coffee/DrainManagerTests.js | 135 ++--- .../test/acceptance/coffee/EarlyDisconnect.js | 299 ++++++----- .../acceptance/coffee/HttpControllerTests.js | 127 +++-- .../test/acceptance/coffee/JoinDocTests.js | 481 ++++++++++------- .../acceptance/coffee/JoinProjectTests.js | 216 +++++--- .../test/acceptance/coffee/LeaveDocTests.js | 173 +++--- .../acceptance/coffee/LeaveProjectTests.js | 285 ++++++---- .../test/acceptance/coffee/PubSubRace.js | 394 ++++++++------ .../acceptance/coffee/ReceiveUpdateTests.js | 410 +++++++++------ .../test/acceptance/coffee/RouterTests.js | 149 +++--- .../acceptance/coffee/SessionSocketsTests.js | 133 +++-- .../test/acceptance/coffee/SessionTests.js | 80 +-- .../coffee/helpers/FixturesManager.js | 87 ++-- .../coffee/helpers/MockDocUpdaterServer.js | 93 ++-- .../coffee/helpers/MockWebServer.js | 90 ++-- .../coffee/helpers/RealTimeClient.js | 133 +++-- .../coffee/helpers/RealtimeServer.js | 65 ++- 19 files changed, 2448 insertions(+), 1664 deletions(-) diff --git a/services/real-time/test/acceptance/coffee/ApplyUpdateTests.js b/services/real-time/test/acceptance/coffee/ApplyUpdateTests.js index f2437f2641..41d1a5e100 100644 --- a/services/real-time/test/acceptance/coffee/ApplyUpdateTests.js +++ b/services/real-time/test/acceptance/coffee/ApplyUpdateTests.js @@ -1,222 +1,317 @@ -async = require "async" -chai = require("chai") -expect = chai.expect -chai.should() +/* + * decaffeinate suggestions: + * DS101: Remove unnecessary use of Array.from + * DS102: Remove unnecessary code created because of implicit returns + * DS201: Simplify complex destructure assignments + * Full docs: https://github.com/decaffeinate/decaffeinate/blob/master/docs/suggestions.md + */ +const async = require("async"); +const chai = require("chai"); +const { + expect +} = chai; +chai.should(); -RealTimeClient = require "./helpers/RealTimeClient" -FixturesManager = require "./helpers/FixturesManager" +const RealTimeClient = require("./helpers/RealTimeClient"); +const FixturesManager = require("./helpers/FixturesManager"); -settings = require "settings-sharelatex" -redis = require "redis-sharelatex" -rclient = redis.createClient(settings.redis.documentupdater) +const settings = require("settings-sharelatex"); +const redis = require("redis-sharelatex"); +const rclient = redis.createClient(settings.redis.documentupdater); -redisSettings = settings.redis +const redisSettings = settings.redis; -describe "applyOtUpdate", -> - before -> - @update = { +describe("applyOtUpdate", function() { + before(function() { + return this.update = { op: [{i: "foo", p: 42}] - } - describe "when authorized", -> - before (done) -> - async.series [ - (cb) => - FixturesManager.setUpProject { + };}); + describe("when authorized", function() { + before(function(done) { + return async.series([ + cb => { + return FixturesManager.setUpProject({ privilegeLevel: "readAndWrite" - }, (e, {@project_id, @user_id}) => - cb(e) + }, (e, {project_id, user_id}) => { + this.project_id = project_id; + this.user_id = user_id; + return cb(e); + }); + }, - (cb) => - FixturesManager.setUpDoc @project_id, {@lines, @version, @ops}, (e, {@doc_id}) => - cb(e) + cb => { + return FixturesManager.setUpDoc(this.project_id, {lines: this.lines, version: this.version, ops: this.ops}, (e, {doc_id}) => { + this.doc_id = doc_id; + return cb(e); + }); + }, - (cb) => - @client = RealTimeClient.connect() - @client.on "connectionAccepted", cb + cb => { + this.client = RealTimeClient.connect(); + return this.client.on("connectionAccepted", cb); + }, - (cb) => - @client.emit "joinProject", project_id: @project_id, cb + cb => { + return this.client.emit("joinProject", {project_id: this.project_id}, cb); + }, - (cb) => - @client.emit "joinDoc", @doc_id, cb + cb => { + return this.client.emit("joinDoc", this.doc_id, cb); + }, - (cb) => - @client.emit "applyOtUpdate", @doc_id, @update, cb - ], done - - it "should push the doc into the pending updates list", (done) -> - rclient.lrange "pending-updates-list", 0, -1, (error, [doc_id]) => - doc_id.should.equal "#{@project_id}:#{@doc_id}" - done() - return null - - it "should push the update into redis", (done) -> - rclient.lrange redisSettings.documentupdater.key_schema.pendingUpdates({@doc_id}), 0, -1, (error, [update]) => - update = JSON.parse(update) - update.op.should.deep.equal @update.op - update.meta.should.deep.equal { - source: @client.publicId - user_id: @user_id + cb => { + return this.client.emit("applyOtUpdate", this.doc_id, this.update, cb); } - done() - return null - - after (done) -> - async.series [ - (cb) => rclient.del "pending-updates-list", cb - (cb) => rclient.del "DocsWithPendingUpdates", "#{@project_id}:#{@doc_id}", cb - (cb) => rclient.del redisSettings.documentupdater.key_schema.pendingUpdates(@doc_id), cb - ], done + ], done); + }); - describe "when authorized with a huge edit update", -> - before (done) -> - @update = { + it("should push the doc into the pending updates list", function(done) { + rclient.lrange("pending-updates-list", 0, -1, (error, ...rest) => { + const [doc_id] = Array.from(rest[0]); + doc_id.should.equal(`${this.project_id}:${this.doc_id}`); + return done(); + }); + return null; + }); + + it("should push the update into redis", function(done) { + rclient.lrange(redisSettings.documentupdater.key_schema.pendingUpdates({doc_id: this.doc_id}), 0, -1, (error, ...rest) => { + let [update] = Array.from(rest[0]); + update = JSON.parse(update); + update.op.should.deep.equal(this.update.op); + update.meta.should.deep.equal({ + source: this.client.publicId, + user_id: this.user_id + }); + return done(); + }); + return null; + }); + + return after(function(done) { + return async.series([ + cb => rclient.del("pending-updates-list", cb), + cb => rclient.del("DocsWithPendingUpdates", `${this.project_id}:${this.doc_id}`, cb), + cb => rclient.del(redisSettings.documentupdater.key_schema.pendingUpdates(this.doc_id), cb) + ], done); + }); + }); + + describe("when authorized with a huge edit update", function() { + before(function(done) { + this.update = { op: { p: 12, - t: "update is too large".repeat(1024 * 400) # >7MB + t: "update is too large".repeat(1024 * 400) // >7MB } - } - async.series [ - (cb) => - FixturesManager.setUpProject { + }; + return async.series([ + cb => { + return FixturesManager.setUpProject({ privilegeLevel: "readAndWrite" - }, (e, {@project_id, @user_id}) => - cb(e) + }, (e, {project_id, user_id}) => { + this.project_id = project_id; + this.user_id = user_id; + return cb(e); + }); + }, - (cb) => - FixturesManager.setUpDoc @project_id, {@lines, @version, @ops}, (e, {@doc_id}) => - cb(e) + cb => { + return FixturesManager.setUpDoc(this.project_id, {lines: this.lines, version: this.version, ops: this.ops}, (e, {doc_id}) => { + this.doc_id = doc_id; + return cb(e); + }); + }, - (cb) => - @client = RealTimeClient.connect() - @client.on "connectionAccepted", cb - @client.on "otUpdateError", (@otUpdateError) => - - (cb) => - @client.emit "joinProject", project_id: @project_id, cb - - (cb) => - @client.emit "joinDoc", @doc_id, cb - - (cb) => - @client.emit "applyOtUpdate", @doc_id, @update, (@error) => - cb() - ], done - - it "should not return an error", -> - expect(@error).to.not.exist - - it "should send an otUpdateError to the client", (done) -> - setTimeout () => - expect(@otUpdateError).to.exist - done() - , 300 - - it "should disconnect the client", (done) -> - setTimeout () => - @client.socket.connected.should.equal false - done() - , 300 - - it "should not put the update in redis", (done) -> - rclient.llen redisSettings.documentupdater.key_schema.pendingUpdates({@doc_id}), (error, len) => - len.should.equal 0 - done() - return null - - describe "when authorized to read-only with an edit update", -> - before (done) -> - async.series [ - (cb) => - FixturesManager.setUpProject { - privilegeLevel: "readOnly" - }, (e, {@project_id, @user_id}) => - cb(e) - - (cb) => - FixturesManager.setUpDoc @project_id, {@lines, @version, @ops}, (e, {@doc_id}) => - cb(e) - - (cb) => - @client = RealTimeClient.connect() - @client.on "connectionAccepted", cb + cb => { + this.client = RealTimeClient.connect(); + this.client.on("connectionAccepted", cb); + return this.client.on("otUpdateError", otUpdateError => { + this.otUpdateError = otUpdateError; - (cb) => - @client.emit "joinProject", project_id: @project_id, cb - - (cb) => - @client.emit "joinDoc", @doc_id, cb - - (cb) => - @client.emit "applyOtUpdate", @doc_id, @update, (@error) => - cb() - ], done - - it "should return an error", -> - expect(@error).to.exist - - it "should disconnect the client", (done) -> - setTimeout () => - @client.socket.connected.should.equal false - done() - , 300 - - it "should not put the update in redis", (done) -> - rclient.llen redisSettings.documentupdater.key_schema.pendingUpdates({@doc_id}), (error, len) => - len.should.equal 0 - done() - return null - - describe "when authorized to read-only with a comment update", -> - before (done) -> - @comment_update = { - op: [{c: "foo", p: 42}] - } - async.series [ - (cb) => - FixturesManager.setUpProject { - privilegeLevel: "readOnly" - }, (e, {@project_id, @user_id}) => - cb(e) - - (cb) => - FixturesManager.setUpDoc @project_id, {@lines, @version, @ops}, (e, {@doc_id}) => - cb(e) + }); + }, - (cb) => - @client = RealTimeClient.connect() - @client.on "connectionAccepted", cb - - (cb) => - @client.emit "joinProject", project_id: @project_id, cb - - (cb) => - @client.emit "joinDoc", @doc_id, cb - - (cb) => - @client.emit "applyOtUpdate", @doc_id, @comment_update, cb - ], done - - it "should push the doc into the pending updates list", (done) -> - rclient.lrange "pending-updates-list", 0, -1, (error, [doc_id]) => - doc_id.should.equal "#{@project_id}:#{@doc_id}" - done() - return null + cb => { + return this.client.emit("joinProject", {project_id: this.project_id}, cb); + }, - it "should push the update into redis", (done) -> - rclient.lrange redisSettings.documentupdater.key_schema.pendingUpdates({@doc_id}), 0, -1, (error, [update]) => - update = JSON.parse(update) - update.op.should.deep.equal @comment_update.op - update.meta.should.deep.equal { - source: @client.publicId - user_id: @user_id + cb => { + return this.client.emit("joinDoc", this.doc_id, cb); + }, + + cb => { + return this.client.emit("applyOtUpdate", this.doc_id, this.update, error => { + this.error = error; + return cb(); + }); } - done() - return null + ], done); + }); - after (done) -> - async.series [ - (cb) => rclient.del "pending-updates-list", cb - (cb) => rclient.del "DocsWithPendingUpdates", "#{@project_id}:#{@doc_id}", cb - (cb) => rclient.del redisSettings.documentupdater.key_schema.pendingUpdates({@doc_id}), cb - ], done + it("should not return an error", function() { + return expect(this.error).to.not.exist; + }); + + it("should send an otUpdateError to the client", function(done) { + return setTimeout(() => { + expect(this.otUpdateError).to.exist; + return done(); + } + , 300); + }); + + it("should disconnect the client", function(done) { + return setTimeout(() => { + this.client.socket.connected.should.equal(false); + return done(); + } + , 300); + }); + + return it("should not put the update in redis", function(done) { + rclient.llen(redisSettings.documentupdater.key_schema.pendingUpdates({doc_id: this.doc_id}), (error, len) => { + len.should.equal(0); + return done(); + }); + return null; + }); + }); + + describe("when authorized to read-only with an edit update", function() { + before(function(done) { + return async.series([ + cb => { + return FixturesManager.setUpProject({ + privilegeLevel: "readOnly" + }, (e, {project_id, user_id}) => { + this.project_id = project_id; + this.user_id = user_id; + return cb(e); + }); + }, + + cb => { + return FixturesManager.setUpDoc(this.project_id, {lines: this.lines, version: this.version, ops: this.ops}, (e, {doc_id}) => { + this.doc_id = doc_id; + return cb(e); + }); + }, + + cb => { + this.client = RealTimeClient.connect(); + return this.client.on("connectionAccepted", cb); + }, + + cb => { + return this.client.emit("joinProject", {project_id: this.project_id}, cb); + }, + + cb => { + return this.client.emit("joinDoc", this.doc_id, cb); + }, + + cb => { + return this.client.emit("applyOtUpdate", this.doc_id, this.update, error => { + this.error = error; + return cb(); + }); + } + ], done); + }); + + it("should return an error", function() { + return expect(this.error).to.exist; + }); + + it("should disconnect the client", function(done) { + return setTimeout(() => { + this.client.socket.connected.should.equal(false); + return done(); + } + , 300); + }); + + return it("should not put the update in redis", function(done) { + rclient.llen(redisSettings.documentupdater.key_schema.pendingUpdates({doc_id: this.doc_id}), (error, len) => { + len.should.equal(0); + return done(); + }); + return null; + }); + }); + + return describe("when authorized to read-only with a comment update", function() { + before(function(done) { + this.comment_update = { + op: [{c: "foo", p: 42}] + }; + return async.series([ + cb => { + return FixturesManager.setUpProject({ + privilegeLevel: "readOnly" + }, (e, {project_id, user_id}) => { + this.project_id = project_id; + this.user_id = user_id; + return cb(e); + }); + }, + + cb => { + return FixturesManager.setUpDoc(this.project_id, {lines: this.lines, version: this.version, ops: this.ops}, (e, {doc_id}) => { + this.doc_id = doc_id; + return cb(e); + }); + }, + + cb => { + this.client = RealTimeClient.connect(); + return this.client.on("connectionAccepted", cb); + }, + + cb => { + return this.client.emit("joinProject", {project_id: this.project_id}, cb); + }, + + cb => { + return this.client.emit("joinDoc", this.doc_id, cb); + }, + + cb => { + return this.client.emit("applyOtUpdate", this.doc_id, this.comment_update, cb); + } + ], done); + }); + + it("should push the doc into the pending updates list", function(done) { + rclient.lrange("pending-updates-list", 0, -1, (error, ...rest) => { + const [doc_id] = Array.from(rest[0]); + doc_id.should.equal(`${this.project_id}:${this.doc_id}`); + return done(); + }); + return null; + }); + + it("should push the update into redis", function(done) { + rclient.lrange(redisSettings.documentupdater.key_schema.pendingUpdates({doc_id: this.doc_id}), 0, -1, (error, ...rest) => { + let [update] = Array.from(rest[0]); + update = JSON.parse(update); + update.op.should.deep.equal(this.comment_update.op); + update.meta.should.deep.equal({ + source: this.client.publicId, + user_id: this.user_id + }); + return done(); + }); + return null; + }); + + return after(function(done) { + return async.series([ + cb => rclient.del("pending-updates-list", cb), + cb => rclient.del("DocsWithPendingUpdates", `${this.project_id}:${this.doc_id}`, cb), + cb => rclient.del(redisSettings.documentupdater.key_schema.pendingUpdates({doc_id: this.doc_id}), cb) + ], done); + }); + }); +}); diff --git a/services/real-time/test/acceptance/coffee/ClientTrackingTests.js b/services/real-time/test/acceptance/coffee/ClientTrackingTests.js index e76eca7d8e..8406aebcbe 100644 --- a/services/real-time/test/acceptance/coffee/ClientTrackingTests.js +++ b/services/real-time/test/acceptance/coffee/ClientTrackingTests.js @@ -1,146 +1,191 @@ -chai = require("chai") -expect = chai.expect -chai.should() +/* + * decaffeinate suggestions: + * DS101: Remove unnecessary use of Array.from + * DS102: Remove unnecessary code created because of implicit returns + * DS207: Consider shorter variations of null checks + * Full docs: https://github.com/decaffeinate/decaffeinate/blob/master/docs/suggestions.md + */ +const chai = require("chai"); +const { + expect +} = chai; +chai.should(); -RealTimeClient = require "./helpers/RealTimeClient" -MockWebServer = require "./helpers/MockWebServer" -FixturesManager = require "./helpers/FixturesManager" +const RealTimeClient = require("./helpers/RealTimeClient"); +const MockWebServer = require("./helpers/MockWebServer"); +const FixturesManager = require("./helpers/FixturesManager"); -async = require "async" +const async = require("async"); -describe "clientTracking", -> - describe "when a client updates its cursor location", -> - before (done) -> - async.series [ - (cb) => - FixturesManager.setUpProject { - privilegeLevel: "owner" +describe("clientTracking", function() { + describe("when a client updates its cursor location", function() { + before(function(done) { + return async.series([ + cb => { + return FixturesManager.setUpProject({ + privilegeLevel: "owner", project: { name: "Test Project" } - }, (error, {@user_id, @project_id}) => cb() + }, (error, {user_id, project_id}) => { this.user_id = user_id; this.project_id = project_id; return cb(); }); + }, - (cb) => - FixturesManager.setUpDoc @project_id, {@lines, @version, @ops}, (e, {@doc_id}) => - cb(e) + cb => { + return FixturesManager.setUpDoc(this.project_id, {lines: this.lines, version: this.version, ops: this.ops}, (e, {doc_id}) => { + this.doc_id = doc_id; + return cb(e); + }); + }, - (cb) => - @clientA = RealTimeClient.connect() - @clientA.on "connectionAccepted", cb + cb => { + this.clientA = RealTimeClient.connect(); + return this.clientA.on("connectionAccepted", cb); + }, - (cb) => - @clientB = RealTimeClient.connect() - @clientB.on "connectionAccepted", cb + cb => { + this.clientB = RealTimeClient.connect(); + return this.clientB.on("connectionAccepted", cb); + }, - (cb) => - @clientA.emit "joinProject", { - project_id: @project_id - }, cb + cb => { + return this.clientA.emit("joinProject", { + project_id: this.project_id + }, cb); + }, - (cb) => - @clientA.emit "joinDoc", @doc_id, cb + cb => { + return this.clientA.emit("joinDoc", this.doc_id, cb); + }, - (cb) => - @clientB.emit "joinProject", { - project_id: @project_id - }, cb + cb => { + return this.clientB.emit("joinProject", { + project_id: this.project_id + }, cb); + }, - (cb) => - @updates = [] - @clientB.on "clientTracking.clientUpdated", (data) => - @updates.push data + cb => { + this.updates = []; + this.clientB.on("clientTracking.clientUpdated", data => { + return this.updates.push(data); + }); - @clientA.emit "clientTracking.updatePosition", { - row: @row = 42 - column: @column = 36 - doc_id: @doc_id - }, (error) -> - throw error if error? - setTimeout cb, 300 # Give the message a chance to reach client B. - ], done + return this.clientA.emit("clientTracking.updatePosition", { + row: (this.row = 42), + column: (this.column = 36), + doc_id: this.doc_id + }, function(error) { + if (error != null) { throw error; } + return setTimeout(cb, 300); + }); + } // Give the message a chance to reach client B. + ], done); + }); - it "should tell other clients about the update", -> - @updates.should.deep.equal [ + it("should tell other clients about the update", function() { + return this.updates.should.deep.equal([ { - row: @row - column: @column - doc_id: @doc_id - id: @clientA.publicId - user_id: @user_id + row: this.row, + column: this.column, + doc_id: this.doc_id, + id: this.clientA.publicId, + user_id: this.user_id, name: "Joe Bloggs" } - ] + ]); + }); - it "should record the update in getConnectedUsers", (done) -> - @clientB.emit "clientTracking.getConnectedUsers", (error, users) => - for user in users - if user.client_id == @clientA.publicId + return it("should record the update in getConnectedUsers", function(done) { + return this.clientB.emit("clientTracking.getConnectedUsers", (error, users) => { + for (let user of Array.from(users)) { + if (user.client_id === this.clientA.publicId) { expect(user.cursorData).to.deep.equal({ - row: @row - column: @column - doc_id: @doc_id - }) - return done() - throw new Error("user was never found") + row: this.row, + column: this.column, + doc_id: this.doc_id + }); + return done(); + } + } + throw new Error("user was never found"); + }); + }); + }); - describe "when an anonymous client updates its cursor location", -> - before (done) -> - async.series [ - (cb) => - FixturesManager.setUpProject { - privilegeLevel: "owner" - project: { name: "Test Project" } + return describe("when an anonymous client updates its cursor location", function() { + before(function(done) { + return async.series([ + cb => { + return FixturesManager.setUpProject({ + privilegeLevel: "owner", + project: { name: "Test Project" }, publicAccess: "readAndWrite" - }, (error, {@user_id, @project_id}) => cb() + }, (error, {user_id, project_id}) => { this.user_id = user_id; this.project_id = project_id; return cb(); }); + }, - (cb) => - FixturesManager.setUpDoc @project_id, {@lines, @version, @ops}, (e, {@doc_id}) => - cb(e) + cb => { + return FixturesManager.setUpDoc(this.project_id, {lines: this.lines, version: this.version, ops: this.ops}, (e, {doc_id}) => { + this.doc_id = doc_id; + return cb(e); + }); + }, - (cb) => - @clientA = RealTimeClient.connect() - @clientA.on "connectionAccepted", cb + cb => { + this.clientA = RealTimeClient.connect(); + return this.clientA.on("connectionAccepted", cb); + }, - (cb) => - @clientA.emit "joinProject", { - project_id: @project_id - }, cb + cb => { + return this.clientA.emit("joinProject", { + project_id: this.project_id + }, cb); + }, - (cb) => - RealTimeClient.setSession({}, cb) + cb => { + return RealTimeClient.setSession({}, cb); + }, - (cb) => - @anonymous = RealTimeClient.connect() - @anonymous.on "connectionAccepted", cb + cb => { + this.anonymous = RealTimeClient.connect(); + return this.anonymous.on("connectionAccepted", cb); + }, - (cb) => - @anonymous.emit "joinProject", { - project_id: @project_id - }, cb + cb => { + return this.anonymous.emit("joinProject", { + project_id: this.project_id + }, cb); + }, - (cb) => - @anonymous.emit "joinDoc", @doc_id, cb + cb => { + return this.anonymous.emit("joinDoc", this.doc_id, cb); + }, - (cb) => - @updates = [] - @clientA.on "clientTracking.clientUpdated", (data) => - @updates.push data + cb => { + this.updates = []; + this.clientA.on("clientTracking.clientUpdated", data => { + return this.updates.push(data); + }); - @anonymous.emit "clientTracking.updatePosition", { - row: @row = 42 - column: @column = 36 - doc_id: @doc_id - }, (error) -> - throw error if error? - setTimeout cb, 300 # Give the message a chance to reach client B. - ], done + return this.anonymous.emit("clientTracking.updatePosition", { + row: (this.row = 42), + column: (this.column = 36), + doc_id: this.doc_id + }, function(error) { + if (error != null) { throw error; } + return setTimeout(cb, 300); + }); + } // Give the message a chance to reach client B. + ], done); + }); - it "should tell other clients about the update", -> - @updates.should.deep.equal [ + return it("should tell other clients about the update", function() { + return this.updates.should.deep.equal([ { - row: @row - column: @column - doc_id: @doc_id - id: @anonymous.publicId - user_id: "anonymous-user" + row: this.row, + column: this.column, + doc_id: this.doc_id, + id: this.anonymous.publicId, + user_id: "anonymous-user", name: "" } - ] + ]); + }); +}); +}); diff --git a/services/real-time/test/acceptance/coffee/DrainManagerTests.js b/services/real-time/test/acceptance/coffee/DrainManagerTests.js index ca967408d8..91bf8836ec 100644 --- a/services/real-time/test/acceptance/coffee/DrainManagerTests.js +++ b/services/real-time/test/acceptance/coffee/DrainManagerTests.js @@ -1,81 +1,100 @@ -RealTimeClient = require "./helpers/RealTimeClient" -FixturesManager = require "./helpers/FixturesManager" +/* + * decaffeinate suggestions: + * DS102: Remove unnecessary code created because of implicit returns + * Full docs: https://github.com/decaffeinate/decaffeinate/blob/master/docs/suggestions.md + */ +const RealTimeClient = require("./helpers/RealTimeClient"); +const FixturesManager = require("./helpers/FixturesManager"); -expect = require("chai").expect +const { + expect +} = require("chai"); -async = require "async" -request = require "request" +const async = require("async"); +const request = require("request"); -Settings = require "settings-sharelatex" +const Settings = require("settings-sharelatex"); -drain = (rate, callback) -> - request.post { - url: "http://localhost:3026/drain?rate=#{rate}" +const drain = function(rate, callback) { + request.post({ + url: `http://localhost:3026/drain?rate=${rate}`, auth: { user: Settings.internal.realTime.user, pass: Settings.internal.realTime.pass } - }, (error, response, data) -> - callback error, data - return null + }, (error, response, data) => callback(error, data)); + return null; +}; -describe "DrainManagerTests", -> - before (done) -> - FixturesManager.setUpProject { - privilegeLevel: "owner" +describe("DrainManagerTests", function() { + before(function(done) { + FixturesManager.setUpProject({ + privilegeLevel: "owner", project: { name: "Test Project" } - }, (e, {@project_id, @user_id}) => done() - return null + }, (e, {project_id, user_id}) => { this.project_id = project_id; this.user_id = user_id; return done(); }); + return null; + }); - before (done) -> - # cleanup to speedup reconnecting - @timeout(10000) - RealTimeClient.disconnectAllClients done + before(function(done) { + // cleanup to speedup reconnecting + this.timeout(10000); + return RealTimeClient.disconnectAllClients(done); + }); - # trigger and check cleanup - it "should have disconnected all previous clients", (done) -> - RealTimeClient.getConnectedClients (error, data) -> - return done(error) if error - expect(data.length).to.equal(0) - done() + // trigger and check cleanup + it("should have disconnected all previous clients", done => RealTimeClient.getConnectedClients(function(error, data) { + if (error) { return done(error); } + expect(data.length).to.equal(0); + return done(); + })); - describe "with two clients in the project", -> - beforeEach (done) -> - async.series [ - (cb) => - @clientA = RealTimeClient.connect() - @clientA.on "connectionAccepted", cb + return describe("with two clients in the project", function() { + beforeEach(function(done) { + return async.series([ + cb => { + this.clientA = RealTimeClient.connect(); + return this.clientA.on("connectionAccepted", cb); + }, - (cb) => - @clientB = RealTimeClient.connect() - @clientB.on "connectionAccepted", cb + cb => { + this.clientB = RealTimeClient.connect(); + return this.clientB.on("connectionAccepted", cb); + }, - (cb) => - @clientA.emit "joinProject", project_id: @project_id, cb + cb => { + return this.clientA.emit("joinProject", {project_id: this.project_id}, cb); + }, - (cb) => - @clientB.emit "joinProject", project_id: @project_id, cb - ], done + cb => { + return this.clientB.emit("joinProject", {project_id: this.project_id}, cb); + } + ], done); + }); - describe "starting to drain", () -> - beforeEach (done) -> - async.parallel [ - (cb) => - @clientA.on "reconnectGracefully", cb - (cb) => - @clientB.on "reconnectGracefully", cb + return describe("starting to drain", function() { + beforeEach(function(done) { + return async.parallel([ + cb => { + return this.clientA.on("reconnectGracefully", cb); + }, + cb => { + return this.clientB.on("reconnectGracefully", cb); + }, - (cb) -> drain(2, cb) - ], done + cb => drain(2, cb) + ], done); + }); - afterEach (done) -> - drain(0, done) # reset drain + afterEach(done => drain(0, done)); // reset drain - it "should not timeout", -> - expect(true).to.equal(true) + it("should not timeout", () => expect(true).to.equal(true)); - it "should not have disconnected", -> - expect(@clientA.socket.connected).to.equal true - expect(@clientB.socket.connected).to.equal true + return it("should not have disconnected", function() { + expect(this.clientA.socket.connected).to.equal(true); + return expect(this.clientB.socket.connected).to.equal(true); + }); + }); + }); +}); diff --git a/services/real-time/test/acceptance/coffee/EarlyDisconnect.js b/services/real-time/test/acceptance/coffee/EarlyDisconnect.js index d90c36b430..875f33ee33 100644 --- a/services/real-time/test/acceptance/coffee/EarlyDisconnect.js +++ b/services/real-time/test/acceptance/coffee/EarlyDisconnect.js @@ -1,160 +1,209 @@ -async = require "async" -{expect} = require("chai") +/* + * decaffeinate suggestions: + * DS101: Remove unnecessary use of Array.from + * DS102: Remove unnecessary code created because of implicit returns + * Full docs: https://github.com/decaffeinate/decaffeinate/blob/master/docs/suggestions.md + */ +const async = require("async"); +const {expect} = require("chai"); -RealTimeClient = require "./helpers/RealTimeClient" -MockDocUpdaterServer = require "./helpers/MockDocUpdaterServer" -MockWebServer = require "./helpers/MockWebServer" -FixturesManager = require "./helpers/FixturesManager" +const RealTimeClient = require("./helpers/RealTimeClient"); +const MockDocUpdaterServer = require("./helpers/MockDocUpdaterServer"); +const MockWebServer = require("./helpers/MockWebServer"); +const FixturesManager = require("./helpers/FixturesManager"); -settings = require "settings-sharelatex" -redis = require "redis-sharelatex" -rclient = redis.createClient(settings.redis.pubsub) -rclientRT = redis.createClient(settings.redis.realtime) -KeysRT = settings.redis.realtime.key_schema +const settings = require("settings-sharelatex"); +const redis = require("redis-sharelatex"); +const rclient = redis.createClient(settings.redis.pubsub); +const rclientRT = redis.createClient(settings.redis.realtime); +const KeysRT = settings.redis.realtime.key_schema; -describe "EarlyDisconnect", -> - before (done) -> - MockDocUpdaterServer.run done +describe("EarlyDisconnect", function() { + before(done => MockDocUpdaterServer.run(done)); - describe "when the client disconnects before joinProject completes", -> - before () -> - # slow down web-api requests to force the race condition - @actualWebAPIjoinProject = joinProject = MockWebServer.joinProject - MockWebServer.joinProject = (project_id, user_id, cb) -> - setTimeout () -> - joinProject(project_id, user_id, cb) - , 300 + describe("when the client disconnects before joinProject completes", function() { + before(function() { + // slow down web-api requests to force the race condition + let joinProject; + this.actualWebAPIjoinProject = (joinProject = MockWebServer.joinProject); + return MockWebServer.joinProject = (project_id, user_id, cb) => setTimeout(() => joinProject(project_id, user_id, cb) + , 300); + }); - after () -> - MockWebServer.joinProject = @actualWebAPIjoinProject + after(function() { + return MockWebServer.joinProject = this.actualWebAPIjoinProject; + }); - beforeEach (done) -> - async.series [ - (cb) => - FixturesManager.setUpProject { - privilegeLevel: "owner" + beforeEach(function(done) { + return async.series([ + cb => { + return FixturesManager.setUpProject({ + privilegeLevel: "owner", project: { name: "Test Project" } - }, (e, {@project_id, @user_id}) => cb() + }, (e, {project_id, user_id}) => { this.project_id = project_id; this.user_id = user_id; return cb(); }); + }, - (cb) => - @clientA = RealTimeClient.connect() - @clientA.on "connectionAccepted", cb + cb => { + this.clientA = RealTimeClient.connect(); + return this.clientA.on("connectionAccepted", cb); + }, - (cb) => - @clientA.emit "joinProject", project_id: @project_id, (() ->) - # disconnect before joinProject completes - @clientA.on "disconnect", () -> cb() - @clientA.disconnect() + cb => { + this.clientA.emit("joinProject", {project_id: this.project_id}, (function() {})); + // disconnect before joinProject completes + this.clientA.on("disconnect", () => cb()); + return this.clientA.disconnect(); + }, - (cb) => - # wait for joinDoc and subscribe - setTimeout cb, 500 - ], done + cb => { + // wait for joinDoc and subscribe + return setTimeout(cb, 500); + } + ], done); + }); - # we can force the race condition, there is no need to repeat too often - for attempt in Array.from(length: 5).map((_, i) -> i+1) - it "should not subscribe to the pub/sub channel anymore (race #{attempt})", (done) -> - rclient.pubsub 'CHANNELS', (err, resp) => - return done(err) if err - expect(resp).to.not.include "editor-events:#{@project_id}" - done() - return null + // we can force the race condition, there is no need to repeat too often + return Array.from(Array.from({length: 5}).map((_, i) => i+1)).map((attempt) => + it(`should not subscribe to the pub/sub channel anymore (race ${attempt})`, function(done) { + rclient.pubsub('CHANNELS', (err, resp) => { + if (err) { return done(err); } + expect(resp).to.not.include(`editor-events:${this.project_id}`); + return done(); + }); + return null; + })); + }); - describe "when the client disconnects before joinDoc completes", -> - beforeEach (done) -> - async.series [ - (cb) => - FixturesManager.setUpProject { - privilegeLevel: "owner" + describe("when the client disconnects before joinDoc completes", function() { + beforeEach(function(done) { + return async.series([ + cb => { + return FixturesManager.setUpProject({ + privilegeLevel: "owner", project: { name: "Test Project" } - }, (e, {@project_id, @user_id}) => cb() + }, (e, {project_id, user_id}) => { this.project_id = project_id; this.user_id = user_id; return cb(); }); + }, - (cb) => - @clientA = RealTimeClient.connect() - @clientA.on "connectionAccepted", cb + cb => { + this.clientA = RealTimeClient.connect(); + return this.clientA.on("connectionAccepted", cb); + }, - (cb) => - @clientA.emit "joinProject", project_id: @project_id, (error, @project, @privilegeLevel, @protocolVersion) => - cb(error) + cb => { + return this.clientA.emit("joinProject", {project_id: this.project_id}, (error, project, privilegeLevel, protocolVersion) => { + this.project = project; + this.privilegeLevel = privilegeLevel; + this.protocolVersion = protocolVersion; + return cb(error); + }); + }, - (cb) => - FixturesManager.setUpDoc @project_id, {@lines, @version, @ops}, (e, {@doc_id}) => - cb(e) + cb => { + return FixturesManager.setUpDoc(this.project_id, {lines: this.lines, version: this.version, ops: this.ops}, (e, {doc_id}) => { + this.doc_id = doc_id; + return cb(e); + }); + }, - (cb) => - @clientA.emit "joinDoc", @doc_id, (() ->) - # disconnect before joinDoc completes - @clientA.on "disconnect", () -> cb() - @clientA.disconnect() + cb => { + this.clientA.emit("joinDoc", this.doc_id, (function() {})); + // disconnect before joinDoc completes + this.clientA.on("disconnect", () => cb()); + return this.clientA.disconnect(); + }, - (cb) => - # wait for subscribe and unsubscribe - setTimeout cb, 100 - ], done + cb => { + // wait for subscribe and unsubscribe + return setTimeout(cb, 100); + } + ], done); + }); - # we can not force the race condition, so we have to try many times - for attempt in Array.from(length: 20).map((_, i) -> i+1) - it "should not subscribe to the pub/sub channels anymore (race #{attempt})", (done) -> - rclient.pubsub 'CHANNELS', (err, resp) => - return done(err) if err - expect(resp).to.not.include "editor-events:#{@project_id}" + // we can not force the race condition, so we have to try many times + return Array.from(Array.from({length: 20}).map((_, i) => i+1)).map((attempt) => + it(`should not subscribe to the pub/sub channels anymore (race ${attempt})`, function(done) { + rclient.pubsub('CHANNELS', (err, resp) => { + if (err) { return done(err); } + expect(resp).to.not.include(`editor-events:${this.project_id}`); - rclient.pubsub 'CHANNELS', (err, resp) => - return done(err) if err - expect(resp).to.not.include "applied-ops:#{@doc_id}" - done() - return null + return rclient.pubsub('CHANNELS', (err, resp) => { + if (err) { return done(err); } + expect(resp).to.not.include(`applied-ops:${this.doc_id}`); + return done(); + }); + }); + return null; + })); + }); - describe "when the client disconnects before clientTracking.updatePosition starts", -> - beforeEach (done) -> - async.series [ - (cb) => - FixturesManager.setUpProject { - privilegeLevel: "owner" + return describe("when the client disconnects before clientTracking.updatePosition starts", function() { + beforeEach(function(done) { + return async.series([ + cb => { + return FixturesManager.setUpProject({ + privilegeLevel: "owner", project: { name: "Test Project" } - }, (e, {@project_id, @user_id}) => cb() + }, (e, {project_id, user_id}) => { this.project_id = project_id; this.user_id = user_id; return cb(); }); + }, - (cb) => - @clientA = RealTimeClient.connect() - @clientA.on "connectionAccepted", cb + cb => { + this.clientA = RealTimeClient.connect(); + return this.clientA.on("connectionAccepted", cb); + }, - (cb) => - @clientA.emit "joinProject", project_id: @project_id, (error, @project, @privilegeLevel, @protocolVersion) => - cb(error) + cb => { + return this.clientA.emit("joinProject", {project_id: this.project_id}, (error, project, privilegeLevel, protocolVersion) => { + this.project = project; + this.privilegeLevel = privilegeLevel; + this.protocolVersion = protocolVersion; + return cb(error); + }); + }, - (cb) => - FixturesManager.setUpDoc @project_id, {@lines, @version, @ops}, (e, {@doc_id}) => - cb(e) + cb => { + return FixturesManager.setUpDoc(this.project_id, {lines: this.lines, version: this.version, ops: this.ops}, (e, {doc_id}) => { + this.doc_id = doc_id; + return cb(e); + }); + }, - (cb) => - @clientA.emit "joinDoc", @doc_id, cb + cb => { + return this.clientA.emit("joinDoc", this.doc_id, cb); + }, - (cb) => - @clientA.emit "clientTracking.updatePosition", { - row: 42 - column: 36 - doc_id: @doc_id - }, (() ->) - # disconnect before updateClientPosition completes - @clientA.on "disconnect", () -> cb() - @clientA.disconnect() + cb => { + this.clientA.emit("clientTracking.updatePosition", { + row: 42, + column: 36, + doc_id: this.doc_id + }, (function() {})); + // disconnect before updateClientPosition completes + this.clientA.on("disconnect", () => cb()); + return this.clientA.disconnect(); + }, - (cb) => - # wait for updateClientPosition - setTimeout cb, 100 - ], done + cb => { + // wait for updateClientPosition + return setTimeout(cb, 100); + } + ], done); + }); - # we can not force the race condition, so we have to try many times - for attempt in Array.from(length: 20).map((_, i) -> i+1) - it "should not show the client as connected (race #{attempt})", (done) -> - rclientRT.smembers KeysRT.clientsInProject({project_id: @project_id}), (err, results) -> - return done(err) if err - expect(results).to.deep.equal([]) - done() - return null + // we can not force the race condition, so we have to try many times + return Array.from(Array.from({length: 20}).map((_, i) => i+1)).map((attempt) => + it(`should not show the client as connected (race ${attempt})`, function(done) { + rclientRT.smembers(KeysRT.clientsInProject({project_id: this.project_id}), function(err, results) { + if (err) { return done(err); } + expect(results).to.deep.equal([]); + return done(); + }); + return null; + })); + }); +}); diff --git a/services/real-time/test/acceptance/coffee/HttpControllerTests.js b/services/real-time/test/acceptance/coffee/HttpControllerTests.js index 524ea7e5de..701b1f7d23 100644 --- a/services/real-time/test/acceptance/coffee/HttpControllerTests.js +++ b/services/real-time/test/acceptance/coffee/HttpControllerTests.js @@ -1,68 +1,91 @@ -async = require('async') -expect = require('chai').expect -request = require('request').defaults({ +/* + * decaffeinate suggestions: + * DS102: Remove unnecessary code created because of implicit returns + * Full docs: https://github.com/decaffeinate/decaffeinate/blob/master/docs/suggestions.md + */ +const async = require('async'); +const { + expect +} = require('chai'); +const request = require('request').defaults({ baseUrl: 'http://localhost:3026' -}) +}); -RealTimeClient = require "./helpers/RealTimeClient" -FixturesManager = require "./helpers/FixturesManager" +const RealTimeClient = require("./helpers/RealTimeClient"); +const FixturesManager = require("./helpers/FixturesManager"); -describe 'HttpControllerTests', -> - describe 'without a user', -> - it 'should return 404 for the client view', (done) -> - client_id = 'not-existing' - request.get { - url: "/clients/#{client_id}" - json: true - }, (error, response, data) -> - return done(error) if error - expect(response.statusCode).to.equal(404) - done() +describe('HttpControllerTests', function() { + describe('without a user', () => it('should return 404 for the client view', function(done) { + const client_id = 'not-existing'; + return request.get({ + url: `/clients/${client_id}`, + json: true + }, function(error, response, data) { + if (error) { return done(error); } + expect(response.statusCode).to.equal(404); + return done(); + }); + })); - describe 'with a user and after joining a project', -> - before (done) -> - async.series [ - (cb) => - FixturesManager.setUpProject { + return describe('with a user and after joining a project', function() { + before(function(done) { + return async.series([ + cb => { + return FixturesManager.setUpProject({ privilegeLevel: "owner" - }, (error, {@project_id, @user_id}) => - cb(error) + }, (error, {project_id, user_id}) => { + this.project_id = project_id; + this.user_id = user_id; + return cb(error); + }); + }, - (cb) => - FixturesManager.setUpDoc @project_id, {}, (error, {@doc_id}) => - cb(error) + cb => { + return FixturesManager.setUpDoc(this.project_id, {}, (error, {doc_id}) => { + this.doc_id = doc_id; + return cb(error); + }); + }, - (cb) => - @client = RealTimeClient.connect() - @client.on "connectionAccepted", cb + cb => { + this.client = RealTimeClient.connect(); + return this.client.on("connectionAccepted", cb); + }, - (cb) => - @client.emit "joinProject", {@project_id}, cb + cb => { + return this.client.emit("joinProject", {project_id: this.project_id}, cb); + }, - (cb) => - @client.emit "joinDoc", @doc_id, cb - ], done + cb => { + return this.client.emit("joinDoc", this.doc_id, cb); + } + ], done); + }); - it 'should send a client view', (done) -> - request.get { - url: "/clients/#{@client.socket.sessionid}" + return it('should send a client view', function(done) { + return request.get({ + url: `/clients/${this.client.socket.sessionid}`, json: true - }, (error, response, data) => - return done(error) if error - expect(response.statusCode).to.equal(200) - expect(data.connected_time).to.exist - delete data.connected_time - # .email is not set in the session - delete data.email + }, (error, response, data) => { + if (error) { return done(error); } + expect(response.statusCode).to.equal(200); + expect(data.connected_time).to.exist; + delete data.connected_time; + // .email is not set in the session + delete data.email; expect(data).to.deep.equal({ - client_id: @client.socket.sessionid, + client_id: this.client.socket.sessionid, first_name: 'Joe', last_name: 'Bloggs', - project_id: @project_id, - user_id: @user_id, + project_id: this.project_id, + user_id: this.user_id, rooms: [ - @project_id, - @doc_id, + this.project_id, + this.doc_id, ] - }) - done() + }); + return done(); + }); + }); + }); +}); diff --git a/services/real-time/test/acceptance/coffee/JoinDocTests.js b/services/real-time/test/acceptance/coffee/JoinDocTests.js index 6c204b6079..0026a6e858 100644 --- a/services/real-time/test/acceptance/coffee/JoinDocTests.js +++ b/services/real-time/test/acceptance/coffee/JoinDocTests.js @@ -1,246 +1,351 @@ -chai = require("chai") -expect = chai.expect -chai.should() +/* + * decaffeinate suggestions: + * DS101: Remove unnecessary use of Array.from + * DS102: Remove unnecessary code created because of implicit returns + * Full docs: https://github.com/decaffeinate/decaffeinate/blob/master/docs/suggestions.md + */ +const chai = require("chai"); +const { + expect +} = chai; +chai.should(); -RealTimeClient = require "./helpers/RealTimeClient" -MockDocUpdaterServer = require "./helpers/MockDocUpdaterServer" -FixturesManager = require "./helpers/FixturesManager" +const RealTimeClient = require("./helpers/RealTimeClient"); +const MockDocUpdaterServer = require("./helpers/MockDocUpdaterServer"); +const FixturesManager = require("./helpers/FixturesManager"); -async = require "async" +const async = require("async"); -describe "joinDoc", -> - before -> - @lines = ["test", "doc", "lines"] - @version = 42 - @ops = ["mock", "doc", "ops"] - @ranges = {"mock": "ranges"} +describe("joinDoc", function() { + before(function() { + this.lines = ["test", "doc", "lines"]; + this.version = 42; + this.ops = ["mock", "doc", "ops"]; + return this.ranges = {"mock": "ranges"};}); - describe "when authorised readAndWrite", -> - before (done) -> - async.series [ - (cb) => - FixturesManager.setUpProject { + describe("when authorised readAndWrite", function() { + before(function(done) { + return async.series([ + cb => { + return FixturesManager.setUpProject({ privilegeLevel: "readAndWrite" - }, (e, {@project_id, @user_id}) => - cb(e) + }, (e, {project_id, user_id}) => { + this.project_id = project_id; + this.user_id = user_id; + return cb(e); + }); + }, - (cb) => - FixturesManager.setUpDoc @project_id, {@lines, @version, @ops, @ranges}, (e, {@doc_id}) => - cb(e) + cb => { + return FixturesManager.setUpDoc(this.project_id, {lines: this.lines, version: this.version, ops: this.ops, ranges: this.ranges}, (e, {doc_id}) => { + this.doc_id = doc_id; + return cb(e); + }); + }, - (cb) => - @client = RealTimeClient.connect() - @client.on "connectionAccepted", cb + cb => { + this.client = RealTimeClient.connect(); + return this.client.on("connectionAccepted", cb); + }, - (cb) => - @client.emit "joinProject", project_id: @project_id, cb + cb => { + return this.client.emit("joinProject", {project_id: this.project_id}, cb); + }, - (cb) => - @client.emit "joinDoc", @doc_id, (error, @returnedArgs...) => cb(error) - ], done + cb => { + return this.client.emit("joinDoc", this.doc_id, (error, ...rest) => { [...this.returnedArgs] = Array.from(rest); return cb(error); }); + } + ], done); + }); - it "should get the doc from the doc updater", -> - MockDocUpdaterServer.getDocument - .calledWith(@project_id, @doc_id, -1) - .should.equal true + it("should get the doc from the doc updater", function() { + return MockDocUpdaterServer.getDocument + .calledWith(this.project_id, this.doc_id, -1) + .should.equal(true); + }); - it "should return the doc lines, version, ranges and ops", -> - @returnedArgs.should.deep.equal [@lines, @version, @ops, @ranges] + it("should return the doc lines, version, ranges and ops", function() { + return this.returnedArgs.should.deep.equal([this.lines, this.version, this.ops, this.ranges]); + }); - it "should have joined the doc room", (done) -> - RealTimeClient.getConnectedClient @client.socket.sessionid, (error, client) => - expect(@doc_id in client.rooms).to.equal true - done() + return it("should have joined the doc room", function(done) { + return RealTimeClient.getConnectedClient(this.client.socket.sessionid, (error, client) => { + expect(Array.from(client.rooms).includes(this.doc_id)).to.equal(true); + return done(); + }); + }); + }); - describe "when authorised readOnly", -> - before (done) -> - async.series [ - (cb) => - FixturesManager.setUpProject { + describe("when authorised readOnly", function() { + before(function(done) { + return async.series([ + cb => { + return FixturesManager.setUpProject({ privilegeLevel: "readOnly" - }, (e, {@project_id, @user_id}) => - cb(e) + }, (e, {project_id, user_id}) => { + this.project_id = project_id; + this.user_id = user_id; + return cb(e); + }); + }, - (cb) => - FixturesManager.setUpDoc @project_id, {@lines, @version, @ops, @ranges}, (e, {@doc_id}) => - cb(e) + cb => { + return FixturesManager.setUpDoc(this.project_id, {lines: this.lines, version: this.version, ops: this.ops, ranges: this.ranges}, (e, {doc_id}) => { + this.doc_id = doc_id; + return cb(e); + }); + }, - (cb) => - @client = RealTimeClient.connect() - @client.on "connectionAccepted", cb + cb => { + this.client = RealTimeClient.connect(); + return this.client.on("connectionAccepted", cb); + }, - (cb) => - @client.emit "joinProject", project_id: @project_id, cb + cb => { + return this.client.emit("joinProject", {project_id: this.project_id}, cb); + }, - (cb) => - @client.emit "joinDoc", @doc_id, (error, @returnedArgs...) => cb(error) - ], done + cb => { + return this.client.emit("joinDoc", this.doc_id, (error, ...rest) => { [...this.returnedArgs] = Array.from(rest); return cb(error); }); + } + ], done); + }); - it "should get the doc from the doc updater", -> - MockDocUpdaterServer.getDocument - .calledWith(@project_id, @doc_id, -1) - .should.equal true + it("should get the doc from the doc updater", function() { + return MockDocUpdaterServer.getDocument + .calledWith(this.project_id, this.doc_id, -1) + .should.equal(true); + }); - it "should return the doc lines, version, ranges and ops", -> - @returnedArgs.should.deep.equal [@lines, @version, @ops, @ranges] + it("should return the doc lines, version, ranges and ops", function() { + return this.returnedArgs.should.deep.equal([this.lines, this.version, this.ops, this.ranges]); + }); - it "should have joined the doc room", (done) -> - RealTimeClient.getConnectedClient @client.socket.sessionid, (error, client) => - expect(@doc_id in client.rooms).to.equal true - done() + return it("should have joined the doc room", function(done) { + return RealTimeClient.getConnectedClient(this.client.socket.sessionid, (error, client) => { + expect(Array.from(client.rooms).includes(this.doc_id)).to.equal(true); + return done(); + }); + }); + }); - describe "when authorised as owner", -> - before (done) -> - async.series [ - (cb) => - FixturesManager.setUpProject { + describe("when authorised as owner", function() { + before(function(done) { + return async.series([ + cb => { + return FixturesManager.setUpProject({ privilegeLevel: "owner" - }, (e, {@project_id, @user_id}) => - cb(e) + }, (e, {project_id, user_id}) => { + this.project_id = project_id; + this.user_id = user_id; + return cb(e); + }); + }, - (cb) => - FixturesManager.setUpDoc @project_id, {@lines, @version, @ops, @ranges}, (e, {@doc_id}) => - cb(e) + cb => { + return FixturesManager.setUpDoc(this.project_id, {lines: this.lines, version: this.version, ops: this.ops, ranges: this.ranges}, (e, {doc_id}) => { + this.doc_id = doc_id; + return cb(e); + }); + }, - (cb) => - @client = RealTimeClient.connect() - @client.on "connectionAccepted", cb + cb => { + this.client = RealTimeClient.connect(); + return this.client.on("connectionAccepted", cb); + }, - (cb) => - @client.emit "joinProject", project_id: @project_id, cb + cb => { + return this.client.emit("joinProject", {project_id: this.project_id}, cb); + }, - (cb) => - @client.emit "joinDoc", @doc_id, (error, @returnedArgs...) => cb(error) - ], done + cb => { + return this.client.emit("joinDoc", this.doc_id, (error, ...rest) => { [...this.returnedArgs] = Array.from(rest); return cb(error); }); + } + ], done); + }); - it "should get the doc from the doc updater", -> - MockDocUpdaterServer.getDocument - .calledWith(@project_id, @doc_id, -1) - .should.equal true + it("should get the doc from the doc updater", function() { + return MockDocUpdaterServer.getDocument + .calledWith(this.project_id, this.doc_id, -1) + .should.equal(true); + }); - it "should return the doc lines, version, ranges and ops", -> - @returnedArgs.should.deep.equal [@lines, @version, @ops, @ranges] + it("should return the doc lines, version, ranges and ops", function() { + return this.returnedArgs.should.deep.equal([this.lines, this.version, this.ops, this.ranges]); + }); - it "should have joined the doc room", (done) -> - RealTimeClient.getConnectedClient @client.socket.sessionid, (error, client) => - expect(@doc_id in client.rooms).to.equal true - done() + return it("should have joined the doc room", function(done) { + return RealTimeClient.getConnectedClient(this.client.socket.sessionid, (error, client) => { + expect(Array.from(client.rooms).includes(this.doc_id)).to.equal(true); + return done(); + }); + }); + }); - # It is impossible to write an acceptance test to test joining an unauthorized - # project, since joinProject already catches that. If you can join a project, - # then you can join a doc in that project. + // It is impossible to write an acceptance test to test joining an unauthorized + // project, since joinProject already catches that. If you can join a project, + // then you can join a doc in that project. - describe "with a fromVersion", -> - before (done) -> - @fromVersion = 36 - async.series [ - (cb) => - FixturesManager.setUpProject { + describe("with a fromVersion", function() { + before(function(done) { + this.fromVersion = 36; + return async.series([ + cb => { + return FixturesManager.setUpProject({ privilegeLevel: "readAndWrite" - }, (e, {@project_id, @user_id}) => - cb(e) + }, (e, {project_id, user_id}) => { + this.project_id = project_id; + this.user_id = user_id; + return cb(e); + }); + }, - (cb) => - FixturesManager.setUpDoc @project_id, {@lines, @version, @ops, @ranges}, (e, {@doc_id}) => - cb(e) + cb => { + return FixturesManager.setUpDoc(this.project_id, {lines: this.lines, version: this.version, ops: this.ops, ranges: this.ranges}, (e, {doc_id}) => { + this.doc_id = doc_id; + return cb(e); + }); + }, - (cb) => - @client = RealTimeClient.connect() - @client.on "connectionAccepted", cb + cb => { + this.client = RealTimeClient.connect(); + return this.client.on("connectionAccepted", cb); + }, - (cb) => - @client.emit "joinProject", project_id: @project_id, cb + cb => { + return this.client.emit("joinProject", {project_id: this.project_id}, cb); + }, - (cb) => - @client.emit "joinDoc", @doc_id, @fromVersion, (error, @returnedArgs...) => cb(error) - ], done + cb => { + return this.client.emit("joinDoc", this.doc_id, this.fromVersion, (error, ...rest) => { [...this.returnedArgs] = Array.from(rest); return cb(error); }); + } + ], done); + }); - it "should get the doc from the doc updater with the fromVersion", -> - MockDocUpdaterServer.getDocument - .calledWith(@project_id, @doc_id, @fromVersion) - .should.equal true + it("should get the doc from the doc updater with the fromVersion", function() { + return MockDocUpdaterServer.getDocument + .calledWith(this.project_id, this.doc_id, this.fromVersion) + .should.equal(true); + }); - it "should return the doc lines, version, ranges and ops", -> - @returnedArgs.should.deep.equal [@lines, @version, @ops, @ranges] + it("should return the doc lines, version, ranges and ops", function() { + return this.returnedArgs.should.deep.equal([this.lines, this.version, this.ops, this.ranges]); + }); - it "should have joined the doc room", (done) -> - RealTimeClient.getConnectedClient @client.socket.sessionid, (error, client) => - expect(@doc_id in client.rooms).to.equal true - done() + return it("should have joined the doc room", function(done) { + return RealTimeClient.getConnectedClient(this.client.socket.sessionid, (error, client) => { + expect(Array.from(client.rooms).includes(this.doc_id)).to.equal(true); + return done(); + }); + }); + }); - describe "with options", -> - before (done) -> - @options = { encodeRanges: true } - async.series [ - (cb) => - FixturesManager.setUpProject { + describe("with options", function() { + before(function(done) { + this.options = { encodeRanges: true }; + return async.series([ + cb => { + return FixturesManager.setUpProject({ privilegeLevel: "readAndWrite" - }, (e, {@project_id, @user_id}) => - cb(e) + }, (e, {project_id, user_id}) => { + this.project_id = project_id; + this.user_id = user_id; + return cb(e); + }); + }, - (cb) => - FixturesManager.setUpDoc @project_id, {@lines, @version, @ops, @ranges}, (e, {@doc_id}) => - cb(e) + cb => { + return FixturesManager.setUpDoc(this.project_id, {lines: this.lines, version: this.version, ops: this.ops, ranges: this.ranges}, (e, {doc_id}) => { + this.doc_id = doc_id; + return cb(e); + }); + }, - (cb) => - @client = RealTimeClient.connect() - @client.on "connectionAccepted", cb + cb => { + this.client = RealTimeClient.connect(); + return this.client.on("connectionAccepted", cb); + }, - (cb) => - @client.emit "joinProject", project_id: @project_id, cb + cb => { + return this.client.emit("joinProject", {project_id: this.project_id}, cb); + }, - (cb) => - @client.emit "joinDoc", @doc_id, @options, (error, @returnedArgs...) => cb(error) - ], done + cb => { + return this.client.emit("joinDoc", this.doc_id, this.options, (error, ...rest) => { [...this.returnedArgs] = Array.from(rest); return cb(error); }); + } + ], done); + }); - it "should get the doc from the doc updater with the default fromVersion", -> - MockDocUpdaterServer.getDocument - .calledWith(@project_id, @doc_id, -1) - .should.equal true + it("should get the doc from the doc updater with the default fromVersion", function() { + return MockDocUpdaterServer.getDocument + .calledWith(this.project_id, this.doc_id, -1) + .should.equal(true); + }); - it "should return the doc lines, version, ranges and ops", -> - @returnedArgs.should.deep.equal [@lines, @version, @ops, @ranges] + it("should return the doc lines, version, ranges and ops", function() { + return this.returnedArgs.should.deep.equal([this.lines, this.version, this.ops, this.ranges]); + }); - it "should have joined the doc room", (done) -> - RealTimeClient.getConnectedClient @client.socket.sessionid, (error, client) => - expect(@doc_id in client.rooms).to.equal true - done() + return it("should have joined the doc room", function(done) { + return RealTimeClient.getConnectedClient(this.client.socket.sessionid, (error, client) => { + expect(Array.from(client.rooms).includes(this.doc_id)).to.equal(true); + return done(); + }); + }); + }); - describe "with fromVersion and options", -> - before (done) -> - @fromVersion = 36 - @options = { encodeRanges: true } - async.series [ - (cb) => - FixturesManager.setUpProject { + return describe("with fromVersion and options", function() { + before(function(done) { + this.fromVersion = 36; + this.options = { encodeRanges: true }; + return async.series([ + cb => { + return FixturesManager.setUpProject({ privilegeLevel: "readAndWrite" - }, (e, {@project_id, @user_id}) => - cb(e) + }, (e, {project_id, user_id}) => { + this.project_id = project_id; + this.user_id = user_id; + return cb(e); + }); + }, - (cb) => - FixturesManager.setUpDoc @project_id, {@lines, @version, @ops, @ranges}, (e, {@doc_id}) => - cb(e) + cb => { + return FixturesManager.setUpDoc(this.project_id, {lines: this.lines, version: this.version, ops: this.ops, ranges: this.ranges}, (e, {doc_id}) => { + this.doc_id = doc_id; + return cb(e); + }); + }, - (cb) => - @client = RealTimeClient.connect() - @client.on "connectionAccepted", cb + cb => { + this.client = RealTimeClient.connect(); + return this.client.on("connectionAccepted", cb); + }, - (cb) => - @client.emit "joinProject", project_id: @project_id, cb + cb => { + return this.client.emit("joinProject", {project_id: this.project_id}, cb); + }, - (cb) => - @client.emit "joinDoc", @doc_id, @fromVersion, @options, (error, @returnedArgs...) => cb(error) - ], done + cb => { + return this.client.emit("joinDoc", this.doc_id, this.fromVersion, this.options, (error, ...rest) => { [...this.returnedArgs] = Array.from(rest); return cb(error); }); + } + ], done); + }); - it "should get the doc from the doc updater with the fromVersion", -> - MockDocUpdaterServer.getDocument - .calledWith(@project_id, @doc_id, @fromVersion) - .should.equal true + it("should get the doc from the doc updater with the fromVersion", function() { + return MockDocUpdaterServer.getDocument + .calledWith(this.project_id, this.doc_id, this.fromVersion) + .should.equal(true); + }); - it "should return the doc lines, version, ranges and ops", -> - @returnedArgs.should.deep.equal [@lines, @version, @ops, @ranges] + it("should return the doc lines, version, ranges and ops", function() { + return this.returnedArgs.should.deep.equal([this.lines, this.version, this.ops, this.ranges]); + }); - it "should have joined the doc room", (done) -> - RealTimeClient.getConnectedClient @client.socket.sessionid, (error, client) => - expect(@doc_id in client.rooms).to.equal true - done() + return it("should have joined the doc room", function(done) { + return RealTimeClient.getConnectedClient(this.client.socket.sessionid, (error, client) => { + expect(Array.from(client.rooms).includes(this.doc_id)).to.equal(true); + return done(); + }); + }); + }); +}); diff --git a/services/real-time/test/acceptance/coffee/JoinProjectTests.js b/services/real-time/test/acceptance/coffee/JoinProjectTests.js index 11082cdb6c..3f962e2c12 100644 --- a/services/real-time/test/acceptance/coffee/JoinProjectTests.js +++ b/services/real-time/test/acceptance/coffee/JoinProjectTests.js @@ -1,108 +1,162 @@ -chai = require("chai") -expect = chai.expect -chai.should() +/* + * decaffeinate suggestions: + * DS101: Remove unnecessary use of Array.from + * DS102: Remove unnecessary code created because of implicit returns + * Full docs: https://github.com/decaffeinate/decaffeinate/blob/master/docs/suggestions.md + */ +const chai = require("chai"); +const { + expect +} = chai; +chai.should(); -RealTimeClient = require "./helpers/RealTimeClient" -MockWebServer = require "./helpers/MockWebServer" -FixturesManager = require "./helpers/FixturesManager" +const RealTimeClient = require("./helpers/RealTimeClient"); +const MockWebServer = require("./helpers/MockWebServer"); +const FixturesManager = require("./helpers/FixturesManager"); -async = require "async" +const async = require("async"); -describe "joinProject", -> - describe "when authorized", -> - before (done) -> - async.series [ - (cb) => - FixturesManager.setUpProject { - privilegeLevel: "owner" +describe("joinProject", function() { + describe("when authorized", function() { + before(function(done) { + return async.series([ + cb => { + return FixturesManager.setUpProject({ + privilegeLevel: "owner", project: { name: "Test Project" } - }, (e, {@project_id, @user_id}) => - cb(e) + }, (e, {project_id, user_id}) => { + this.project_id = project_id; + this.user_id = user_id; + return cb(e); + }); + }, - (cb) => - @client = RealTimeClient.connect() - @client.on "connectionAccepted", cb + cb => { + this.client = RealTimeClient.connect(); + return this.client.on("connectionAccepted", cb); + }, - (cb) => - @client.emit "joinProject", project_id: @project_id, (error, @project, @privilegeLevel, @protocolVersion) => - cb(error) - ], done + cb => { + return this.client.emit("joinProject", {project_id: this.project_id}, (error, project, privilegeLevel, protocolVersion) => { + this.project = project; + this.privilegeLevel = privilegeLevel; + this.protocolVersion = protocolVersion; + return cb(error); + }); + } + ], done); + }); - it "should get the project from web", -> - MockWebServer.joinProject - .calledWith(@project_id, @user_id) - .should.equal true + it("should get the project from web", function() { + return MockWebServer.joinProject + .calledWith(this.project_id, this.user_id) + .should.equal(true); + }); - it "should return the project", -> - @project.should.deep.equal { + it("should return the project", function() { + return this.project.should.deep.equal({ name: "Test Project" - } + }); + }); - it "should return the privilege level", -> - @privilegeLevel.should.equal "owner" + it("should return the privilege level", function() { + return this.privilegeLevel.should.equal("owner"); + }); - it "should return the protocolVersion", -> - @protocolVersion.should.equal 2 + it("should return the protocolVersion", function() { + return this.protocolVersion.should.equal(2); + }); - it "should have joined the project room", (done) -> - RealTimeClient.getConnectedClient @client.socket.sessionid, (error, client) => - expect(@project_id in client.rooms).to.equal true - done() + it("should have joined the project room", function(done) { + return RealTimeClient.getConnectedClient(this.client.socket.sessionid, (error, client) => { + expect(Array.from(client.rooms).includes(this.project_id)).to.equal(true); + return done(); + }); + }); - it "should have marked the user as connected", (done) -> - @client.emit "clientTracking.getConnectedUsers", (error, users) => - connected = false - for user in users - if user.client_id == @client.publicId and user.user_id == @user_id - connected = true - break - expect(connected).to.equal true - done() + return it("should have marked the user as connected", function(done) { + return this.client.emit("clientTracking.getConnectedUsers", (error, users) => { + let connected = false; + for (let user of Array.from(users)) { + if ((user.client_id === this.client.publicId) && (user.user_id === this.user_id)) { + connected = true; + break; + } + } + expect(connected).to.equal(true); + return done(); + }); + }); + }); - describe "when not authorized", -> - before (done) -> - async.series [ - (cb) => - FixturesManager.setUpProject { - privilegeLevel: null + describe("when not authorized", function() { + before(function(done) { + return async.series([ + cb => { + return FixturesManager.setUpProject({ + privilegeLevel: null, project: { name: "Test Project" } - }, (e, {@project_id, @user_id}) => - cb(e) + }, (e, {project_id, user_id}) => { + this.project_id = project_id; + this.user_id = user_id; + return cb(e); + }); + }, - (cb) => - @client = RealTimeClient.connect() - @client.on "connectionAccepted", cb + cb => { + this.client = RealTimeClient.connect(); + return this.client.on("connectionAccepted", cb); + }, - (cb) => - @client.emit "joinProject", project_id: @project_id, (@error, @project, @privilegeLevel, @protocolVersion) => - cb() - ], done + cb => { + return this.client.emit("joinProject", {project_id: this.project_id}, (error, project, privilegeLevel, protocolVersion) => { + this.error = error; + this.project = project; + this.privilegeLevel = privilegeLevel; + this.protocolVersion = protocolVersion; + return cb(); + }); + } + ], done); + }); - it "should return an error", -> - @error.message.should.equal "not authorized" + it("should return an error", function() { + return this.error.message.should.equal("not authorized"); + }); - it "should not have joined the project room", (done) -> - RealTimeClient.getConnectedClient @client.socket.sessionid, (error, client) => - expect(@project_id in client.rooms).to.equal false - done() + return it("should not have joined the project room", function(done) { + return RealTimeClient.getConnectedClient(this.client.socket.sessionid, (error, client) => { + expect(Array.from(client.rooms).includes(this.project_id)).to.equal(false); + return done(); + }); + }); + }); - describe "when over rate limit", -> - before (done) -> - async.series [ - (cb) => - @client = RealTimeClient.connect() - @client.on "connectionAccepted", cb + return describe("when over rate limit", function() { + before(function(done) { + return async.series([ + cb => { + this.client = RealTimeClient.connect(); + return this.client.on("connectionAccepted", cb); + }, - (cb) => - @client.emit "joinProject", project_id: 'rate-limited', (@error) => - cb() - ], done + cb => { + return this.client.emit("joinProject", {project_id: 'rate-limited'}, error => { + this.error = error; + return cb(); + }); + } + ], done); + }); - it "should return a TooManyRequests error code", -> - @error.message.should.equal "rate-limit hit when joining project" - @error.code.should.equal "TooManyRequests" + return it("should return a TooManyRequests error code", function() { + this.error.message.should.equal("rate-limit hit when joining project"); + return this.error.code.should.equal("TooManyRequests"); + }); + }); +}); diff --git a/services/real-time/test/acceptance/coffee/LeaveDocTests.js b/services/real-time/test/acceptance/coffee/LeaveDocTests.js index e35e9093d3..5e589356f9 100644 --- a/services/real-time/test/acceptance/coffee/LeaveDocTests.js +++ b/services/real-time/test/acceptance/coffee/LeaveDocTests.js @@ -1,86 +1,121 @@ -chai = require("chai") -expect = chai.expect -chai.should() -sinon = require("sinon") +/* + * decaffeinate suggestions: + * DS101: Remove unnecessary use of Array.from + * DS102: Remove unnecessary code created because of implicit returns + * DS207: Consider shorter variations of null checks + * Full docs: https://github.com/decaffeinate/decaffeinate/blob/master/docs/suggestions.md + */ +const chai = require("chai"); +const { + expect +} = chai; +chai.should(); +const sinon = require("sinon"); -RealTimeClient = require "./helpers/RealTimeClient" -MockDocUpdaterServer = require "./helpers/MockDocUpdaterServer" -FixturesManager = require "./helpers/FixturesManager" -logger = require("logger-sharelatex") +const RealTimeClient = require("./helpers/RealTimeClient"); +const MockDocUpdaterServer = require("./helpers/MockDocUpdaterServer"); +const FixturesManager = require("./helpers/FixturesManager"); +const logger = require("logger-sharelatex"); -async = require "async" +const async = require("async"); -describe "leaveDoc", -> - before -> - @lines = ["test", "doc", "lines"] - @version = 42 - @ops = ["mock", "doc", "ops"] - sinon.spy(logger, "error") - sinon.spy(logger, "warn") - sinon.spy(logger, "log") - @other_doc_id = FixturesManager.getRandomId() +describe("leaveDoc", function() { + before(function() { + this.lines = ["test", "doc", "lines"]; + this.version = 42; + this.ops = ["mock", "doc", "ops"]; + sinon.spy(logger, "error"); + sinon.spy(logger, "warn"); + sinon.spy(logger, "log"); + return this.other_doc_id = FixturesManager.getRandomId(); + }); - after -> - logger.error.restore() # remove the spy - logger.warn.restore() - logger.log.restore() + after(function() { + logger.error.restore(); // remove the spy + logger.warn.restore(); + return logger.log.restore(); + }); - describe "when joined to a doc", -> - beforeEach (done) -> - async.series [ - (cb) => - FixturesManager.setUpProject { + return describe("when joined to a doc", function() { + beforeEach(function(done) { + return async.series([ + cb => { + return FixturesManager.setUpProject({ privilegeLevel: "readAndWrite" - }, (e, {@project_id, @user_id}) => - cb(e) + }, (e, {project_id, user_id}) => { + this.project_id = project_id; + this.user_id = user_id; + return cb(e); + }); + }, - (cb) => - FixturesManager.setUpDoc @project_id, {@lines, @version, @ops}, (e, {@doc_id}) => - cb(e) + cb => { + return FixturesManager.setUpDoc(this.project_id, {lines: this.lines, version: this.version, ops: this.ops}, (e, {doc_id}) => { + this.doc_id = doc_id; + return cb(e); + }); + }, - (cb) => - @client = RealTimeClient.connect() - @client.on "connectionAccepted", cb + cb => { + this.client = RealTimeClient.connect(); + return this.client.on("connectionAccepted", cb); + }, - (cb) => - @client.emit "joinProject", project_id: @project_id, cb + cb => { + return this.client.emit("joinProject", {project_id: this.project_id}, cb); + }, - (cb) => - @client.emit "joinDoc", @doc_id, (error, @returnedArgs...) => cb(error) - ], done + cb => { + return this.client.emit("joinDoc", this.doc_id, (error, ...rest) => { [...this.returnedArgs] = Array.from(rest); return cb(error); }); + } + ], done); + }); - describe "then leaving the doc", -> - beforeEach (done) -> - @client.emit "leaveDoc", @doc_id, (error) -> - throw error if error? - done() + describe("then leaving the doc", function() { + beforeEach(function(done) { + return this.client.emit("leaveDoc", this.doc_id, function(error) { + if (error != null) { throw error; } + return done(); + }); + }); - it "should have left the doc room", (done) -> - RealTimeClient.getConnectedClient @client.socket.sessionid, (error, client) => - expect(@doc_id in client.rooms).to.equal false - done() + return it("should have left the doc room", function(done) { + return RealTimeClient.getConnectedClient(this.client.socket.sessionid, (error, client) => { + expect(Array.from(client.rooms).includes(this.doc_id)).to.equal(false); + return done(); + }); + }); + }); - describe "when sending a leaveDoc request before the previous joinDoc request has completed", -> - beforeEach (done) -> - @client.emit "leaveDoc", @doc_id, () -> - @client.emit "joinDoc", @doc_id, () -> - @client.emit "leaveDoc", @doc_id, (error) -> - throw error if error? - done() + describe("when sending a leaveDoc request before the previous joinDoc request has completed", function() { + beforeEach(function(done) { + this.client.emit("leaveDoc", this.doc_id, function() {}); + this.client.emit("joinDoc", this.doc_id, function() {}); + return this.client.emit("leaveDoc", this.doc_id, function(error) { + if (error != null) { throw error; } + return done(); + }); + }); - it "should not trigger an error", -> - sinon.assert.neverCalledWith(logger.error, sinon.match.any, "not subscribed - shouldn't happen") + it("should not trigger an error", () => sinon.assert.neverCalledWith(logger.error, sinon.match.any, "not subscribed - shouldn't happen")); - it "should have left the doc room", (done) -> - RealTimeClient.getConnectedClient @client.socket.sessionid, (error, client) => - expect(@doc_id in client.rooms).to.equal false - done() + return it("should have left the doc room", function(done) { + return RealTimeClient.getConnectedClient(this.client.socket.sessionid, (error, client) => { + expect(Array.from(client.rooms).includes(this.doc_id)).to.equal(false); + return done(); + }); + }); + }); - describe "when sending a leaveDoc for a room the client has not joined ", -> - beforeEach (done) -> - @client.emit "leaveDoc", @other_doc_id, (error) -> - throw error if error? - done() + return describe("when sending a leaveDoc for a room the client has not joined ", function() { + beforeEach(function(done) { + return this.client.emit("leaveDoc", this.other_doc_id, function(error) { + if (error != null) { throw error; } + return done(); + }); + }); - it "should trigger a low level message only", -> - sinon.assert.calledWith(logger.log, sinon.match.any, "ignoring request from client to leave room it is not in") + return it("should trigger a low level message only", () => sinon.assert.calledWith(logger.log, sinon.match.any, "ignoring request from client to leave room it is not in")); + }); + }); +}); diff --git a/services/real-time/test/acceptance/coffee/LeaveProjectTests.js b/services/real-time/test/acceptance/coffee/LeaveProjectTests.js index 91ec1a1159..11e4ed2471 100644 --- a/services/real-time/test/acceptance/coffee/LeaveProjectTests.js +++ b/services/real-time/test/acceptance/coffee/LeaveProjectTests.js @@ -1,147 +1,206 @@ -RealTimeClient = require "./helpers/RealTimeClient" -MockDocUpdaterServer = require "./helpers/MockDocUpdaterServer" -FixturesManager = require "./helpers/FixturesManager" +/* + * decaffeinate suggestions: + * DS101: Remove unnecessary use of Array.from + * DS102: Remove unnecessary code created because of implicit returns + * Full docs: https://github.com/decaffeinate/decaffeinate/blob/master/docs/suggestions.md + */ +const RealTimeClient = require("./helpers/RealTimeClient"); +const MockDocUpdaterServer = require("./helpers/MockDocUpdaterServer"); +const FixturesManager = require("./helpers/FixturesManager"); -async = require "async" +const async = require("async"); -settings = require "settings-sharelatex" -redis = require "redis-sharelatex" -rclient = redis.createClient(settings.redis.pubsub) +const settings = require("settings-sharelatex"); +const redis = require("redis-sharelatex"); +const rclient = redis.createClient(settings.redis.pubsub); -describe "leaveProject", -> - before (done) -> - MockDocUpdaterServer.run done +describe("leaveProject", function() { + before(done => MockDocUpdaterServer.run(done)); - describe "with other clients in the project", -> - before (done) -> - async.series [ - (cb) => - FixturesManager.setUpProject { - privilegeLevel: "owner" + describe("with other clients in the project", function() { + before(function(done) { + return async.series([ + cb => { + return FixturesManager.setUpProject({ + privilegeLevel: "owner", project: { name: "Test Project" } - }, (e, {@project_id, @user_id}) => cb() + }, (e, {project_id, user_id}) => { this.project_id = project_id; this.user_id = user_id; return cb(); }); + }, - (cb) => - @clientA = RealTimeClient.connect() - @clientA.on "connectionAccepted", cb + cb => { + this.clientA = RealTimeClient.connect(); + return this.clientA.on("connectionAccepted", cb); + }, - (cb) => - @clientB = RealTimeClient.connect() - @clientB.on "connectionAccepted", cb + cb => { + this.clientB = RealTimeClient.connect(); + this.clientB.on("connectionAccepted", cb); - @clientBDisconnectMessages = [] - @clientB.on "clientTracking.clientDisconnected", (data) => - @clientBDisconnectMessages.push data + this.clientBDisconnectMessages = []; + return this.clientB.on("clientTracking.clientDisconnected", data => { + return this.clientBDisconnectMessages.push(data); + }); + }, - (cb) => - @clientA.emit "joinProject", project_id: @project_id, (error, @project, @privilegeLevel, @protocolVersion) => - cb(error) + cb => { + return this.clientA.emit("joinProject", {project_id: this.project_id}, (error, project, privilegeLevel, protocolVersion) => { + this.project = project; + this.privilegeLevel = privilegeLevel; + this.protocolVersion = protocolVersion; + return cb(error); + }); + }, - (cb) => - @clientB.emit "joinProject", project_id: @project_id, (error, @project, @privilegeLevel, @protocolVersion) => - cb(error) + cb => { + return this.clientB.emit("joinProject", {project_id: this.project_id}, (error, project, privilegeLevel, protocolVersion) => { + this.project = project; + this.privilegeLevel = privilegeLevel; + this.protocolVersion = protocolVersion; + return cb(error); + }); + }, - (cb) => - FixturesManager.setUpDoc @project_id, {@lines, @version, @ops}, (e, {@doc_id}) => - cb(e) + cb => { + return FixturesManager.setUpDoc(this.project_id, {lines: this.lines, version: this.version, ops: this.ops}, (e, {doc_id}) => { + this.doc_id = doc_id; + return cb(e); + }); + }, - (cb) => - @clientA.emit "joinDoc", @doc_id, cb - (cb) => - @clientB.emit "joinDoc", @doc_id, cb + cb => { + return this.clientA.emit("joinDoc", this.doc_id, cb); + }, + cb => { + return this.clientB.emit("joinDoc", this.doc_id, cb); + }, - (cb) => - # leaveProject is called when the client disconnects - @clientA.on "disconnect", () -> cb() - @clientA.disconnect() + cb => { + // leaveProject is called when the client disconnects + this.clientA.on("disconnect", () => cb()); + return this.clientA.disconnect(); + }, - (cb) => - # The API waits a little while before flushing changes - setTimeout done, 1000 + cb => { + // The API waits a little while before flushing changes + return setTimeout(done, 1000); + } - ], done + ], done); + }); - it "should emit a disconnect message to the room", -> - @clientBDisconnectMessages.should.deep.equal [@clientA.publicId] + it("should emit a disconnect message to the room", function() { + return this.clientBDisconnectMessages.should.deep.equal([this.clientA.publicId]); + }); - it "should no longer list the client in connected users", (done) -> - @clientB.emit "clientTracking.getConnectedUsers", (error, users) => - for user in users - if user.client_id == @clientA.publicId - throw "Expected clientA to not be listed in connected users" - return done() + it("should no longer list the client in connected users", function(done) { + return this.clientB.emit("clientTracking.getConnectedUsers", (error, users) => { + for (let user of Array.from(users)) { + if (user.client_id === this.clientA.publicId) { + throw "Expected clientA to not be listed in connected users"; + } + } + return done(); + }); + }); - it "should not flush the project to the document updater", -> - MockDocUpdaterServer.deleteProject - .calledWith(@project_id) - .should.equal false + it("should not flush the project to the document updater", function() { + return MockDocUpdaterServer.deleteProject + .calledWith(this.project_id) + .should.equal(false); + }); - it "should remain subscribed to the editor-events channels", (done) -> - rclient.pubsub 'CHANNELS', (err, resp) => - return done(err) if err - resp.should.include "editor-events:#{@project_id}" - done() - return null + it("should remain subscribed to the editor-events channels", function(done) { + rclient.pubsub('CHANNELS', (err, resp) => { + if (err) { return done(err); } + resp.should.include(`editor-events:${this.project_id}`); + return done(); + }); + return null; + }); - it "should remain subscribed to the applied-ops channels", (done) -> - rclient.pubsub 'CHANNELS', (err, resp) => - return done(err) if err - resp.should.include "applied-ops:#{@doc_id}" - done() - return null + return it("should remain subscribed to the applied-ops channels", function(done) { + rclient.pubsub('CHANNELS', (err, resp) => { + if (err) { return done(err); } + resp.should.include(`applied-ops:${this.doc_id}`); + return done(); + }); + return null; + }); + }); - describe "with no other clients in the project", -> - before (done) -> - async.series [ - (cb) => - FixturesManager.setUpProject { - privilegeLevel: "owner" + return describe("with no other clients in the project", function() { + before(function(done) { + return async.series([ + cb => { + return FixturesManager.setUpProject({ + privilegeLevel: "owner", project: { name: "Test Project" } - }, (e, {@project_id, @user_id}) => cb() + }, (e, {project_id, user_id}) => { this.project_id = project_id; this.user_id = user_id; return cb(); }); + }, - (cb) => - @clientA = RealTimeClient.connect() - @clientA.on "connect", cb + cb => { + this.clientA = RealTimeClient.connect(); + return this.clientA.on("connect", cb); + }, - (cb) => - @clientA.emit "joinProject", project_id: @project_id, (error, @project, @privilegeLevel, @protocolVersion) => - cb(error) + cb => { + return this.clientA.emit("joinProject", {project_id: this.project_id}, (error, project, privilegeLevel, protocolVersion) => { + this.project = project; + this.privilegeLevel = privilegeLevel; + this.protocolVersion = protocolVersion; + return cb(error); + }); + }, - (cb) => - FixturesManager.setUpDoc @project_id, {@lines, @version, @ops}, (e, {@doc_id}) => - cb(e) - (cb) => - @clientA.emit "joinDoc", @doc_id, cb + cb => { + return FixturesManager.setUpDoc(this.project_id, {lines: this.lines, version: this.version, ops: this.ops}, (e, {doc_id}) => { + this.doc_id = doc_id; + return cb(e); + }); + }, + cb => { + return this.clientA.emit("joinDoc", this.doc_id, cb); + }, - (cb) => - # leaveProject is called when the client disconnects - @clientA.on "disconnect", () -> cb() - @clientA.disconnect() + cb => { + // leaveProject is called when the client disconnects + this.clientA.on("disconnect", () => cb()); + return this.clientA.disconnect(); + }, - (cb) => - # The API waits a little while before flushing changes - setTimeout done, 1000 - ], done + cb => { + // The API waits a little while before flushing changes + return setTimeout(done, 1000); + } + ], done); + }); - it "should flush the project to the document updater", -> - MockDocUpdaterServer.deleteProject - .calledWith(@project_id) - .should.equal true + it("should flush the project to the document updater", function() { + return MockDocUpdaterServer.deleteProject + .calledWith(this.project_id) + .should.equal(true); + }); - it "should not subscribe to the editor-events channels anymore", (done) -> - rclient.pubsub 'CHANNELS', (err, resp) => - return done(err) if err - resp.should.not.include "editor-events:#{@project_id}" - done() - return null + it("should not subscribe to the editor-events channels anymore", function(done) { + rclient.pubsub('CHANNELS', (err, resp) => { + if (err) { return done(err); } + resp.should.not.include(`editor-events:${this.project_id}`); + return done(); + }); + return null; + }); - it "should not subscribe to the applied-ops channels anymore", (done) -> - rclient.pubsub 'CHANNELS', (err, resp) => - return done(err) if err - resp.should.not.include "applied-ops:#{@doc_id}" - done() - return null + return it("should not subscribe to the applied-ops channels anymore", function(done) { + rclient.pubsub('CHANNELS', (err, resp) => { + if (err) { return done(err); } + resp.should.not.include(`applied-ops:${this.doc_id}`); + return done(); + }); + return null; + }); + }); +}); diff --git a/services/real-time/test/acceptance/coffee/PubSubRace.js b/services/real-time/test/acceptance/coffee/PubSubRace.js index d5e6653fac..3c5a6f0669 100644 --- a/services/real-time/test/acceptance/coffee/PubSubRace.js +++ b/services/real-time/test/acceptance/coffee/PubSubRace.js @@ -1,205 +1,277 @@ -RealTimeClient = require "./helpers/RealTimeClient" -MockDocUpdaterServer = require "./helpers/MockDocUpdaterServer" -FixturesManager = require "./helpers/FixturesManager" +/* + * decaffeinate suggestions: + * DS102: Remove unnecessary code created because of implicit returns + * Full docs: https://github.com/decaffeinate/decaffeinate/blob/master/docs/suggestions.md + */ +const RealTimeClient = require("./helpers/RealTimeClient"); +const MockDocUpdaterServer = require("./helpers/MockDocUpdaterServer"); +const FixturesManager = require("./helpers/FixturesManager"); -async = require "async" +const async = require("async"); -settings = require "settings-sharelatex" -redis = require "redis-sharelatex" -rclient = redis.createClient(settings.redis.pubsub) +const settings = require("settings-sharelatex"); +const redis = require("redis-sharelatex"); +const rclient = redis.createClient(settings.redis.pubsub); -describe "PubSubRace", -> - before (done) -> - MockDocUpdaterServer.run done +describe("PubSubRace", function() { + before(done => MockDocUpdaterServer.run(done)); - describe "when the client leaves a doc before joinDoc completes", -> - before (done) -> - async.series [ - (cb) => - FixturesManager.setUpProject { - privilegeLevel: "owner" + describe("when the client leaves a doc before joinDoc completes", function() { + before(function(done) { + return async.series([ + cb => { + return FixturesManager.setUpProject({ + privilegeLevel: "owner", project: { name: "Test Project" } - }, (e, {@project_id, @user_id}) => cb() + }, (e, {project_id, user_id}) => { this.project_id = project_id; this.user_id = user_id; return cb(); }); + }, - (cb) => - @clientA = RealTimeClient.connect() - @clientA.on "connect", cb + cb => { + this.clientA = RealTimeClient.connect(); + return this.clientA.on("connect", cb); + }, - (cb) => - @clientA.emit "joinProject", project_id: @project_id, (error, @project, @privilegeLevel, @protocolVersion) => - cb(error) + cb => { + return this.clientA.emit("joinProject", {project_id: this.project_id}, (error, project, privilegeLevel, protocolVersion) => { + this.project = project; + this.privilegeLevel = privilegeLevel; + this.protocolVersion = protocolVersion; + return cb(error); + }); + }, - (cb) => - FixturesManager.setUpDoc @project_id, {@lines, @version, @ops}, (e, {@doc_id}) => - cb(e) + cb => { + return FixturesManager.setUpDoc(this.project_id, {lines: this.lines, version: this.version, ops: this.ops}, (e, {doc_id}) => { + this.doc_id = doc_id; + return cb(e); + }); + }, - (cb) => - @clientA.emit "joinDoc", @doc_id, () -> - # leave before joinDoc completes - @clientA.emit "leaveDoc", @doc_id, cb + cb => { + this.clientA.emit("joinDoc", this.doc_id, function() {}); + // leave before joinDoc completes + return this.clientA.emit("leaveDoc", this.doc_id, cb); + }, - (cb) => - # wait for subscribe and unsubscribe - setTimeout cb, 100 - ], done + cb => { + // wait for subscribe and unsubscribe + return setTimeout(cb, 100); + } + ], done); + }); - it "should not subscribe to the applied-ops channels anymore", (done) -> - rclient.pubsub 'CHANNELS', (err, resp) => - return done(err) if err - resp.should.not.include "applied-ops:#{@doc_id}" - done() - return null + return it("should not subscribe to the applied-ops channels anymore", function(done) { + rclient.pubsub('CHANNELS', (err, resp) => { + if (err) { return done(err); } + resp.should.not.include(`applied-ops:${this.doc_id}`); + return done(); + }); + return null; + }); + }); - describe "when the client emits joinDoc and leaveDoc requests frequently and leaves eventually", -> - before (done) -> - async.series [ - (cb) => - FixturesManager.setUpProject { - privilegeLevel: "owner" + describe("when the client emits joinDoc and leaveDoc requests frequently and leaves eventually", function() { + before(function(done) { + return async.series([ + cb => { + return FixturesManager.setUpProject({ + privilegeLevel: "owner", project: { name: "Test Project" } - }, (e, {@project_id, @user_id}) => cb() + }, (e, {project_id, user_id}) => { this.project_id = project_id; this.user_id = user_id; return cb(); }); + }, - (cb) => - @clientA = RealTimeClient.connect() - @clientA.on "connect", cb + cb => { + this.clientA = RealTimeClient.connect(); + return this.clientA.on("connect", cb); + }, - (cb) => - @clientA.emit "joinProject", project_id: @project_id, (error, @project, @privilegeLevel, @protocolVersion) => - cb(error) + cb => { + return this.clientA.emit("joinProject", {project_id: this.project_id}, (error, project, privilegeLevel, protocolVersion) => { + this.project = project; + this.privilegeLevel = privilegeLevel; + this.protocolVersion = protocolVersion; + return cb(error); + }); + }, - (cb) => - FixturesManager.setUpDoc @project_id, {@lines, @version, @ops}, (e, {@doc_id}) => - cb(e) + cb => { + return FixturesManager.setUpDoc(this.project_id, {lines: this.lines, version: this.version, ops: this.ops}, (e, {doc_id}) => { + this.doc_id = doc_id; + return cb(e); + }); + }, - (cb) => - @clientA.emit "joinDoc", @doc_id, () -> - @clientA.emit "leaveDoc", @doc_id, () -> - @clientA.emit "joinDoc", @doc_id, () -> - @clientA.emit "leaveDoc", @doc_id, () -> - @clientA.emit "joinDoc", @doc_id, () -> - @clientA.emit "leaveDoc", @doc_id, () -> - @clientA.emit "joinDoc", @doc_id, () -> - @clientA.emit "leaveDoc", @doc_id, () -> - @clientA.emit "joinDoc", @doc_id, () -> - @clientA.emit "leaveDoc", @doc_id, cb + cb => { + this.clientA.emit("joinDoc", this.doc_id, function() {}); + this.clientA.emit("leaveDoc", this.doc_id, function() {}); + this.clientA.emit("joinDoc", this.doc_id, function() {}); + this.clientA.emit("leaveDoc", this.doc_id, function() {}); + this.clientA.emit("joinDoc", this.doc_id, function() {}); + this.clientA.emit("leaveDoc", this.doc_id, function() {}); + this.clientA.emit("joinDoc", this.doc_id, function() {}); + this.clientA.emit("leaveDoc", this.doc_id, function() {}); + this.clientA.emit("joinDoc", this.doc_id, function() {}); + return this.clientA.emit("leaveDoc", this.doc_id, cb); + }, - (cb) => - # wait for subscribe and unsubscribe - setTimeout cb, 100 - ], done + cb => { + // wait for subscribe and unsubscribe + return setTimeout(cb, 100); + } + ], done); + }); - it "should not subscribe to the applied-ops channels anymore", (done) -> - rclient.pubsub 'CHANNELS', (err, resp) => - return done(err) if err - resp.should.not.include "applied-ops:#{@doc_id}" - done() - return null + return it("should not subscribe to the applied-ops channels anymore", function(done) { + rclient.pubsub('CHANNELS', (err, resp) => { + if (err) { return done(err); } + resp.should.not.include(`applied-ops:${this.doc_id}`); + return done(); + }); + return null; + }); + }); - describe "when the client emits joinDoc and leaveDoc requests frequently and remains in the doc", -> - before (done) -> - async.series [ - (cb) => - FixturesManager.setUpProject { - privilegeLevel: "owner" + describe("when the client emits joinDoc and leaveDoc requests frequently and remains in the doc", function() { + before(function(done) { + return async.series([ + cb => { + return FixturesManager.setUpProject({ + privilegeLevel: "owner", project: { name: "Test Project" } - }, (e, {@project_id, @user_id}) => cb() + }, (e, {project_id, user_id}) => { this.project_id = project_id; this.user_id = user_id; return cb(); }); + }, - (cb) => - @clientA = RealTimeClient.connect() - @clientA.on "connect", cb + cb => { + this.clientA = RealTimeClient.connect(); + return this.clientA.on("connect", cb); + }, - (cb) => - @clientA.emit "joinProject", project_id: @project_id, (error, @project, @privilegeLevel, @protocolVersion) => - cb(error) + cb => { + return this.clientA.emit("joinProject", {project_id: this.project_id}, (error, project, privilegeLevel, protocolVersion) => { + this.project = project; + this.privilegeLevel = privilegeLevel; + this.protocolVersion = protocolVersion; + return cb(error); + }); + }, - (cb) => - FixturesManager.setUpDoc @project_id, {@lines, @version, @ops}, (e, {@doc_id}) => - cb(e) + cb => { + return FixturesManager.setUpDoc(this.project_id, {lines: this.lines, version: this.version, ops: this.ops}, (e, {doc_id}) => { + this.doc_id = doc_id; + return cb(e); + }); + }, - (cb) => - @clientA.emit "joinDoc", @doc_id, () -> - @clientA.emit "leaveDoc", @doc_id, () -> - @clientA.emit "joinDoc", @doc_id, () -> - @clientA.emit "leaveDoc", @doc_id, () -> - @clientA.emit "joinDoc", @doc_id, () -> - @clientA.emit "leaveDoc", @doc_id, () -> - @clientA.emit "joinDoc", @doc_id, () -> - @clientA.emit "leaveDoc", @doc_id, () -> - @clientA.emit "joinDoc", @doc_id, cb + cb => { + this.clientA.emit("joinDoc", this.doc_id, function() {}); + this.clientA.emit("leaveDoc", this.doc_id, function() {}); + this.clientA.emit("joinDoc", this.doc_id, function() {}); + this.clientA.emit("leaveDoc", this.doc_id, function() {}); + this.clientA.emit("joinDoc", this.doc_id, function() {}); + this.clientA.emit("leaveDoc", this.doc_id, function() {}); + this.clientA.emit("joinDoc", this.doc_id, function() {}); + this.clientA.emit("leaveDoc", this.doc_id, function() {}); + return this.clientA.emit("joinDoc", this.doc_id, cb); + }, - (cb) => - # wait for subscribe and unsubscribe - setTimeout cb, 100 - ], done + cb => { + // wait for subscribe and unsubscribe + return setTimeout(cb, 100); + } + ], done); + }); - it "should subscribe to the applied-ops channels", (done) -> - rclient.pubsub 'CHANNELS', (err, resp) => - return done(err) if err - resp.should.include "applied-ops:#{@doc_id}" - done() - return null + return it("should subscribe to the applied-ops channels", function(done) { + rclient.pubsub('CHANNELS', (err, resp) => { + if (err) { return done(err); } + resp.should.include(`applied-ops:${this.doc_id}`); + return done(); + }); + return null; + }); + }); - describe "when the client disconnects before joinDoc completes", -> - before (done) -> - async.series [ - (cb) => - FixturesManager.setUpProject { - privilegeLevel: "owner" + return describe("when the client disconnects before joinDoc completes", function() { + before(function(done) { + return async.series([ + cb => { + return FixturesManager.setUpProject({ + privilegeLevel: "owner", project: { name: "Test Project" } - }, (e, {@project_id, @user_id}) => cb() + }, (e, {project_id, user_id}) => { this.project_id = project_id; this.user_id = user_id; return cb(); }); + }, - (cb) => - @clientA = RealTimeClient.connect() - @clientA.on "connect", cb + cb => { + this.clientA = RealTimeClient.connect(); + return this.clientA.on("connect", cb); + }, - (cb) => - @clientA.emit "joinProject", project_id: @project_id, (error, @project, @privilegeLevel, @protocolVersion) => - cb(error) + cb => { + return this.clientA.emit("joinProject", {project_id: this.project_id}, (error, project, privilegeLevel, protocolVersion) => { + this.project = project; + this.privilegeLevel = privilegeLevel; + this.protocolVersion = protocolVersion; + return cb(error); + }); + }, - (cb) => - FixturesManager.setUpDoc @project_id, {@lines, @version, @ops}, (e, {@doc_id}) => - cb(e) + cb => { + return FixturesManager.setUpDoc(this.project_id, {lines: this.lines, version: this.version, ops: this.ops}, (e, {doc_id}) => { + this.doc_id = doc_id; + return cb(e); + }); + }, - (cb) => - joinDocCompleted = false - @clientA.emit "joinDoc", @doc_id, () -> - joinDocCompleted = true - # leave before joinDoc completes - setTimeout () => - if joinDocCompleted - return cb(new Error('joinDocCompleted -- lower timeout')) - @clientA.on "disconnect", () -> cb() - @clientA.disconnect() - # socket.io processes joinDoc and disconnect with different delays: - # - joinDoc goes through two process.nextTick - # - disconnect goes through one process.nextTick - # We have to inject the disconnect event into a different event loop - # cycle. - , 3 + cb => { + let joinDocCompleted = false; + this.clientA.emit("joinDoc", this.doc_id, () => joinDocCompleted = true); + // leave before joinDoc completes + return setTimeout(() => { + if (joinDocCompleted) { + return cb(new Error('joinDocCompleted -- lower timeout')); + } + this.clientA.on("disconnect", () => cb()); + return this.clientA.disconnect(); + } + // socket.io processes joinDoc and disconnect with different delays: + // - joinDoc goes through two process.nextTick + // - disconnect goes through one process.nextTick + // We have to inject the disconnect event into a different event loop + // cycle. + , 3); + }, - (cb) => - # wait for subscribe and unsubscribe - setTimeout cb, 100 - ], done + cb => { + // wait for subscribe and unsubscribe + return setTimeout(cb, 100); + } + ], done); + }); - it "should not subscribe to the editor-events channels anymore", (done) -> - rclient.pubsub 'CHANNELS', (err, resp) => - return done(err) if err - resp.should.not.include "editor-events:#{@project_id}" - done() - return null + it("should not subscribe to the editor-events channels anymore", function(done) { + rclient.pubsub('CHANNELS', (err, resp) => { + if (err) { return done(err); } + resp.should.not.include(`editor-events:${this.project_id}`); + return done(); + }); + return null; + }); - it "should not subscribe to the applied-ops channels anymore", (done) -> - rclient.pubsub 'CHANNELS', (err, resp) => - return done(err) if err - resp.should.not.include "applied-ops:#{@doc_id}" - done() - return null + return it("should not subscribe to the applied-ops channels anymore", function(done) { + rclient.pubsub('CHANNELS', (err, resp) => { + if (err) { return done(err); } + resp.should.not.include(`applied-ops:${this.doc_id}`); + return done(); + }); + return null; + }); + }); +}); diff --git a/services/real-time/test/acceptance/coffee/ReceiveUpdateTests.js b/services/real-time/test/acceptance/coffee/ReceiveUpdateTests.js index da9ee0ca36..960ddb145e 100644 --- a/services/real-time/test/acceptance/coffee/ReceiveUpdateTests.js +++ b/services/real-time/test/acceptance/coffee/ReceiveUpdateTests.js @@ -1,208 +1,274 @@ -chai = require("chai") -expect = chai.expect -chai.should() +/* + * decaffeinate suggestions: + * DS102: Remove unnecessary code created because of implicit returns + * DS207: Consider shorter variations of null checks + * Full docs: https://github.com/decaffeinate/decaffeinate/blob/master/docs/suggestions.md + */ +const chai = require("chai"); +const { + expect +} = chai; +chai.should(); -RealTimeClient = require "./helpers/RealTimeClient" -MockWebServer = require "./helpers/MockWebServer" -FixturesManager = require "./helpers/FixturesManager" +const RealTimeClient = require("./helpers/RealTimeClient"); +const MockWebServer = require("./helpers/MockWebServer"); +const FixturesManager = require("./helpers/FixturesManager"); -async = require "async" +const async = require("async"); -settings = require "settings-sharelatex" -redis = require "redis-sharelatex" -rclient = redis.createClient(settings.redis.pubsub) +const settings = require("settings-sharelatex"); +const redis = require("redis-sharelatex"); +const rclient = redis.createClient(settings.redis.pubsub); -describe "receiveUpdate", -> - beforeEach (done) -> - @lines = ["test", "doc", "lines"] - @version = 42 - @ops = ["mock", "doc", "ops"] +describe("receiveUpdate", function() { + beforeEach(function(done) { + this.lines = ["test", "doc", "lines"]; + this.version = 42; + this.ops = ["mock", "doc", "ops"]; - async.series [ - (cb) => - FixturesManager.setUpProject { - privilegeLevel: "owner" + return async.series([ + cb => { + return FixturesManager.setUpProject({ + privilegeLevel: "owner", project: { name: "Test Project" } - }, (error, {@user_id, @project_id}) => cb() + }, (error, {user_id, project_id}) => { this.user_id = user_id; this.project_id = project_id; return cb(); }); + }, - (cb) => - FixturesManager.setUpDoc @project_id, {@lines, @version, @ops}, (e, {@doc_id}) => - cb(e) + cb => { + return FixturesManager.setUpDoc(this.project_id, {lines: this.lines, version: this.version, ops: this.ops}, (e, {doc_id}) => { + this.doc_id = doc_id; + return cb(e); + }); + }, - (cb) => - @clientA = RealTimeClient.connect() - @clientA.on "connectionAccepted", cb + cb => { + this.clientA = RealTimeClient.connect(); + return this.clientA.on("connectionAccepted", cb); + }, - (cb) => - @clientB = RealTimeClient.connect() - @clientB.on "connectionAccepted", cb + cb => { + this.clientB = RealTimeClient.connect(); + return this.clientB.on("connectionAccepted", cb); + }, - (cb) => - @clientA.emit "joinProject", { - project_id: @project_id - }, cb + cb => { + return this.clientA.emit("joinProject", { + project_id: this.project_id + }, cb); + }, - (cb) => - @clientA.emit "joinDoc", @doc_id, cb + cb => { + return this.clientA.emit("joinDoc", this.doc_id, cb); + }, - (cb) => - @clientB.emit "joinProject", { - project_id: @project_id - }, cb + cb => { + return this.clientB.emit("joinProject", { + project_id: this.project_id + }, cb); + }, - (cb) => - @clientB.emit "joinDoc", @doc_id, cb + cb => { + return this.clientB.emit("joinDoc", this.doc_id, cb); + }, - (cb) => - FixturesManager.setUpProject { - privilegeLevel: "owner" + cb => { + return FixturesManager.setUpProject({ + privilegeLevel: "owner", project: {name: "Test Project"} - }, (error, {user_id: @user_id_second, project_id: @project_id_second}) => cb() + }, (error, {user_id: user_id_second, project_id: project_id_second}) => { this.user_id_second = user_id_second; this.project_id_second = project_id_second; return cb(); }); + }, - (cb) => - FixturesManager.setUpDoc @project_id_second, {@lines, @version, @ops}, (e, {doc_id: @doc_id_second}) => - cb(e) + cb => { + return FixturesManager.setUpDoc(this.project_id_second, {lines: this.lines, version: this.version, ops: this.ops}, (e, {doc_id: doc_id_second}) => { + this.doc_id_second = doc_id_second; + return cb(e); + }); + }, - (cb) => - @clientC = RealTimeClient.connect() - @clientC.on "connectionAccepted", cb + cb => { + this.clientC = RealTimeClient.connect(); + return this.clientC.on("connectionAccepted", cb); + }, - (cb) => - @clientC.emit "joinProject", { - project_id: @project_id_second - }, cb - (cb) => - @clientC.emit "joinDoc", @doc_id_second, cb + cb => { + return this.clientC.emit("joinProject", { + project_id: this.project_id_second + }, cb); + }, + cb => { + return this.clientC.emit("joinDoc", this.doc_id_second, cb); + }, - (cb) => - @clientAUpdates = [] - @clientA.on "otUpdateApplied", (update) => @clientAUpdates.push(update) - @clientBUpdates = [] - @clientB.on "otUpdateApplied", (update) => @clientBUpdates.push(update) - @clientCUpdates = [] - @clientC.on "otUpdateApplied", (update) => @clientCUpdates.push(update) + cb => { + this.clientAUpdates = []; + this.clientA.on("otUpdateApplied", update => this.clientAUpdates.push(update)); + this.clientBUpdates = []; + this.clientB.on("otUpdateApplied", update => this.clientBUpdates.push(update)); + this.clientCUpdates = []; + this.clientC.on("otUpdateApplied", update => this.clientCUpdates.push(update)); - @clientAErrors = [] - @clientA.on "otUpdateError", (error) => @clientAErrors.push(error) - @clientBErrors = [] - @clientB.on "otUpdateError", (error) => @clientBErrors.push(error) - @clientCErrors = [] - @clientC.on "otUpdateError", (error) => @clientCErrors.push(error) - cb() - ], done - - afterEach () -> - @clientA?.disconnect() - @clientB?.disconnect() - @clientC?.disconnect() - - describe "with an update from clientA", -> - beforeEach (done) -> - @update = { - doc_id: @doc_id - op: - meta: - source: @clientA.publicId - v: @version - doc: @doc_id - op: [{i: "foo", p: 50}] + this.clientAErrors = []; + this.clientA.on("otUpdateError", error => this.clientAErrors.push(error)); + this.clientBErrors = []; + this.clientB.on("otUpdateError", error => this.clientBErrors.push(error)); + this.clientCErrors = []; + this.clientC.on("otUpdateError", error => this.clientCErrors.push(error)); + return cb(); } - rclient.publish "applied-ops", JSON.stringify(@update) - setTimeout done, 200 # Give clients time to get message - - it "should send the full op to clientB", -> - @clientBUpdates.should.deep.equal [@update.op] - - it "should send an ack to clientA", -> - @clientAUpdates.should.deep.equal [{ - v: @version, doc: @doc_id - }] + ], done); + }); - it "should send nothing to clientC", -> - @clientCUpdates.should.deep.equal [] + afterEach(function() { + if (this.clientA != null) { + this.clientA.disconnect(); + } + if (this.clientB != null) { + this.clientB.disconnect(); + } + return (this.clientC != null ? this.clientC.disconnect() : undefined); + }); - describe "with an update from clientC", -> - beforeEach (done) -> - @update = { - doc_id: @doc_id_second - op: - meta: - source: @clientC.publicId - v: @version - doc: @doc_id_second - op: [{i: "update from clientC", p: 50}] - } - rclient.publish "applied-ops", JSON.stringify(@update) - setTimeout done, 200 # Give clients time to get message - - it "should send nothing to clientA", -> - @clientAUpdates.should.deep.equal [] - - it "should send nothing to clientB", -> - @clientBUpdates.should.deep.equal [] - - it "should send an ack to clientC", -> - @clientCUpdates.should.deep.equal [{ - v: @version, doc: @doc_id_second - }] - - describe "with an update from a remote client for project 1", -> - beforeEach (done) -> - @update = { - doc_id: @doc_id - op: - meta: - source: 'this-is-a-remote-client-id' - v: @version - doc: @doc_id + describe("with an update from clientA", function() { + beforeEach(function(done) { + this.update = { + doc_id: this.doc_id, + op: { + meta: { + source: this.clientA.publicId + }, + v: this.version, + doc: this.doc_id, op: [{i: "foo", p: 50}] - } - rclient.publish "applied-ops", JSON.stringify(@update) - setTimeout done, 200 # Give clients time to get message - - it "should send the full op to clientA", -> - @clientAUpdates.should.deep.equal [@update.op] + } + }; + rclient.publish("applied-ops", JSON.stringify(this.update)); + return setTimeout(done, 200); + }); // Give clients time to get message - it "should send the full op to clientB", -> - @clientBUpdates.should.deep.equal [@update.op] + it("should send the full op to clientB", function() { + return this.clientBUpdates.should.deep.equal([this.update.op]); + }); + + it("should send an ack to clientA", function() { + return this.clientAUpdates.should.deep.equal([{ + v: this.version, doc: this.doc_id + }]); + }); - it "should send nothing to clientC", -> - @clientCUpdates.should.deep.equal [] + return it("should send nothing to clientC", function() { + return this.clientCUpdates.should.deep.equal([]); + }); +}); - describe "with an error for the first project", -> - beforeEach (done) -> - rclient.publish "applied-ops", JSON.stringify({doc_id: @doc_id, error: @error = "something went wrong"}) - setTimeout done, 200 # Give clients time to get message + describe("with an update from clientC", function() { + beforeEach(function(done) { + this.update = { + doc_id: this.doc_id_second, + op: { + meta: { + source: this.clientC.publicId + }, + v: this.version, + doc: this.doc_id_second, + op: [{i: "update from clientC", p: 50}] + } + }; + rclient.publish("applied-ops", JSON.stringify(this.update)); + return setTimeout(done, 200); + }); // Give clients time to get message - it "should send the error to the clients in the first project", -> - @clientAErrors.should.deep.equal [@error] - @clientBErrors.should.deep.equal [@error] + it("should send nothing to clientA", function() { + return this.clientAUpdates.should.deep.equal([]); + }); - it "should not send any errors to the client in the second project", -> - @clientCErrors.should.deep.equal [] + it("should send nothing to clientB", function() { + return this.clientBUpdates.should.deep.equal([]); + }); - it "should disconnect the clients of the first project", -> - @clientA.socket.connected.should.equal false - @clientB.socket.connected.should.equal false + return it("should send an ack to clientC", function() { + return this.clientCUpdates.should.deep.equal([{ + v: this.version, doc: this.doc_id_second + }]); + }); +}); - it "should not disconnect the client in the second project", -> - @clientC.socket.connected.should.equal true + describe("with an update from a remote client for project 1", function() { + beforeEach(function(done) { + this.update = { + doc_id: this.doc_id, + op: { + meta: { + source: 'this-is-a-remote-client-id' + }, + v: this.version, + doc: this.doc_id, + op: [{i: "foo", p: 50}] + } + }; + rclient.publish("applied-ops", JSON.stringify(this.update)); + return setTimeout(done, 200); + }); // Give clients time to get message - describe "with an error for the second project", -> - beforeEach (done) -> - rclient.publish "applied-ops", JSON.stringify({doc_id: @doc_id_second, error: @error = "something went wrong"}) - setTimeout done, 200 # Give clients time to get message + it("should send the full op to clientA", function() { + return this.clientAUpdates.should.deep.equal([this.update.op]); + }); + + it("should send the full op to clientB", function() { + return this.clientBUpdates.should.deep.equal([this.update.op]); + }); - it "should not send any errors to the clients in the first project", -> - @clientAErrors.should.deep.equal [] - @clientBErrors.should.deep.equal [] + return it("should send nothing to clientC", function() { + return this.clientCUpdates.should.deep.equal([]); + }); +}); - it "should send the error to the client in the second project", -> - @clientCErrors.should.deep.equal [@error] + describe("with an error for the first project", function() { + beforeEach(function(done) { + rclient.publish("applied-ops", JSON.stringify({doc_id: this.doc_id, error: (this.error = "something went wrong")})); + return setTimeout(done, 200); + }); // Give clients time to get message - it "should not disconnect the clients of the first project", -> - @clientA.socket.connected.should.equal true - @clientB.socket.connected.should.equal true + it("should send the error to the clients in the first project", function() { + this.clientAErrors.should.deep.equal([this.error]); + return this.clientBErrors.should.deep.equal([this.error]); + }); - it "should disconnect the client in the second project", -> - @clientC.socket.connected.should.equal false + it("should not send any errors to the client in the second project", function() { + return this.clientCErrors.should.deep.equal([]); + }); + + it("should disconnect the clients of the first project", function() { + this.clientA.socket.connected.should.equal(false); + return this.clientB.socket.connected.should.equal(false); + }); + + return it("should not disconnect the client in the second project", function() { + return this.clientC.socket.connected.should.equal(true); + }); + }); + + return describe("with an error for the second project", function() { + beforeEach(function(done) { + rclient.publish("applied-ops", JSON.stringify({doc_id: this.doc_id_second, error: (this.error = "something went wrong")})); + return setTimeout(done, 200); + }); // Give clients time to get message + + it("should not send any errors to the clients in the first project", function() { + this.clientAErrors.should.deep.equal([]); + return this.clientBErrors.should.deep.equal([]); + }); + + it("should send the error to the client in the second project", function() { + return this.clientCErrors.should.deep.equal([this.error]); + }); + + it("should not disconnect the clients of the first project", function() { + this.clientA.socket.connected.should.equal(true); + return this.clientB.socket.connected.should.equal(true); + }); + + return it("should disconnect the client in the second project", function() { + return this.clientC.socket.connected.should.equal(false); + }); + }); +}); diff --git a/services/real-time/test/acceptance/coffee/RouterTests.js b/services/real-time/test/acceptance/coffee/RouterTests.js index c3952a2887..6254eb5208 100644 --- a/services/real-time/test/acceptance/coffee/RouterTests.js +++ b/services/real-time/test/acceptance/coffee/RouterTests.js @@ -1,76 +1,101 @@ -async = require "async" -{expect} = require("chai") +/* + * decaffeinate suggestions: + * DS102: Remove unnecessary code created because of implicit returns + * Full docs: https://github.com/decaffeinate/decaffeinate/blob/master/docs/suggestions.md + */ +const async = require("async"); +const {expect} = require("chai"); -RealTimeClient = require "./helpers/RealTimeClient" -FixturesManager = require "./helpers/FixturesManager" +const RealTimeClient = require("./helpers/RealTimeClient"); +const FixturesManager = require("./helpers/FixturesManager"); -describe "Router", -> - describe "joinProject", -> - describe "when there is no callback provided", -> - after () -> - process.removeListener('unhandledRejection', @onUnhandled) +describe("Router", () => describe("joinProject", function() { + describe("when there is no callback provided", function() { + after(function() { + return process.removeListener('unhandledRejection', this.onUnhandled); + }); - before (done) -> - @onUnhandled = (error) -> - done(error) - process.on('unhandledRejection', @onUnhandled) - async.series [ - (cb) => - FixturesManager.setUpProject { - privilegeLevel: "owner" - project: { - name: "Test Project" - } - }, (e, {@project_id, @user_id}) => - cb(e) + before(function(done) { + this.onUnhandled = error => done(error); + process.on('unhandledRejection', this.onUnhandled); + return async.series([ + cb => { + return FixturesManager.setUpProject({ + privilegeLevel: "owner", + project: { + name: "Test Project" + } + }, (e, {project_id, user_id}) => { + this.project_id = project_id; + this.user_id = user_id; + return cb(e); + }); + }, - (cb) => - @client = RealTimeClient.connect() - @client.on "connectionAccepted", cb + cb => { + this.client = RealTimeClient.connect(); + return this.client.on("connectionAccepted", cb); + }, - (cb) => - @client = RealTimeClient.connect() - @client.on "connectionAccepted", cb + cb => { + this.client = RealTimeClient.connect(); + return this.client.on("connectionAccepted", cb); + }, - (cb) => - @client.emit "joinProject", project_id: @project_id - setTimeout(cb, 100) - ], done + cb => { + this.client.emit("joinProject", {project_id: this.project_id}); + return setTimeout(cb, 100); + } + ], done); + }); - it "should keep on going", -> - expect('still running').to.exist + return it("should keep on going", () => expect('still running').to.exist); + }); - describe "when there are too many arguments", -> - after () -> - process.removeListener('unhandledRejection', @onUnhandled) + return describe("when there are too many arguments", function() { + after(function() { + return process.removeListener('unhandledRejection', this.onUnhandled); + }); - before (done) -> - @onUnhandled = (error) -> - done(error) - process.on('unhandledRejection', @onUnhandled) - async.series [ - (cb) => - FixturesManager.setUpProject { - privilegeLevel: "owner" - project: { - name: "Test Project" - } - }, (e, {@project_id, @user_id}) => - cb(e) + before(function(done) { + this.onUnhandled = error => done(error); + process.on('unhandledRejection', this.onUnhandled); + return async.series([ + cb => { + return FixturesManager.setUpProject({ + privilegeLevel: "owner", + project: { + name: "Test Project" + } + }, (e, {project_id, user_id}) => { + this.project_id = project_id; + this.user_id = user_id; + return cb(e); + }); + }, - (cb) => - @client = RealTimeClient.connect() - @client.on "connectionAccepted", cb + cb => { + this.client = RealTimeClient.connect(); + return this.client.on("connectionAccepted", cb); + }, - (cb) => - @client = RealTimeClient.connect() - @client.on "connectionAccepted", cb + cb => { + this.client = RealTimeClient.connect(); + return this.client.on("connectionAccepted", cb); + }, - (cb) => - @client.emit "joinProject", 1, 2, 3, 4, 5, (@error) => - cb() - ], done + cb => { + return this.client.emit("joinProject", 1, 2, 3, 4, 5, error => { + this.error = error; + return cb(); + }); + } + ], done); + }); - it "should return an error message", -> - expect(@error.message).to.equal('unexpected arguments') + return it("should return an error message", function() { + return expect(this.error.message).to.equal('unexpected arguments'); + }); + }); +})); diff --git a/services/real-time/test/acceptance/coffee/SessionSocketsTests.js b/services/real-time/test/acceptance/coffee/SessionSocketsTests.js index 3009da682f..912e0912e5 100644 --- a/services/real-time/test/acceptance/coffee/SessionSocketsTests.js +++ b/services/real-time/test/acceptance/coffee/SessionSocketsTests.js @@ -1,67 +1,90 @@ -RealTimeClient = require("./helpers/RealTimeClient") -Settings = require("settings-sharelatex") -{expect} = require('chai') +/* + * decaffeinate suggestions: + * DS102: Remove unnecessary code created because of implicit returns + * Full docs: https://github.com/decaffeinate/decaffeinate/blob/master/docs/suggestions.md + */ +const RealTimeClient = require("./helpers/RealTimeClient"); +const Settings = require("settings-sharelatex"); +const {expect} = require('chai'); -describe 'SessionSockets', -> - before -> - @checkSocket = (fn) -> - client = RealTimeClient.connect() - client.on 'connectionAccepted', fn - client.on 'connectionRejected', fn - return null +describe('SessionSockets', function() { + before(function() { + return this.checkSocket = function(fn) { + const client = RealTimeClient.connect(); + client.on('connectionAccepted', fn); + client.on('connectionRejected', fn); + return null; + }; + }); - describe 'without cookies', -> - before -> - RealTimeClient.cookie = null + describe('without cookies', function() { + before(() => RealTimeClient.cookie = null); - it 'should return a lookup error', (done) -> - @checkSocket (error) -> - expect(error).to.exist - expect(error.message).to.equal('invalid session') - done() + return it('should return a lookup error', function(done) { + return this.checkSocket(function(error) { + expect(error).to.exist; + expect(error.message).to.equal('invalid session'); + return done(); + }); + }); + }); - describe 'with a different cookie', -> - before -> - RealTimeClient.cookie = "some.key=someValue" + describe('with a different cookie', function() { + before(() => RealTimeClient.cookie = "some.key=someValue"); - it 'should return a lookup error', (done) -> - @checkSocket (error) -> - expect(error).to.exist - expect(error.message).to.equal('invalid session') - done() + return it('should return a lookup error', function(done) { + return this.checkSocket(function(error) { + expect(error).to.exist; + expect(error.message).to.equal('invalid session'); + return done(); + }); + }); + }); - describe 'with an invalid cookie', -> - before (done) -> - RealTimeClient.setSession {}, (error) -> - return done(error) if error - RealTimeClient.cookie = "#{Settings.cookieName}=#{ + describe('with an invalid cookie', function() { + before(function(done) { + RealTimeClient.setSession({}, function(error) { + if (error) { return done(error); } + RealTimeClient.cookie = `${Settings.cookieName}=${ RealTimeClient.cookie.slice(17, 49) - }" - done() - return null + }`; + return done(); + }); + return null; + }); - it 'should return a lookup error', (done) -> - @checkSocket (error) -> - expect(error).to.exist - expect(error.message).to.equal('invalid session') - done() + return it('should return a lookup error', function(done) { + return this.checkSocket(function(error) { + expect(error).to.exist; + expect(error.message).to.equal('invalid session'); + return done(); + }); + }); + }); - describe 'with a valid cookie and no matching session', -> - before -> - RealTimeClient.cookie = "#{Settings.cookieName}=unknownId" + describe('with a valid cookie and no matching session', function() { + before(() => RealTimeClient.cookie = `${Settings.cookieName}=unknownId`); - it 'should return a lookup error', (done) -> - @checkSocket (error) -> - expect(error).to.exist - expect(error.message).to.equal('invalid session') - done() + return it('should return a lookup error', function(done) { + return this.checkSocket(function(error) { + expect(error).to.exist; + expect(error.message).to.equal('invalid session'); + return done(); + }); + }); + }); - describe 'with a valid cookie and a matching session', -> - before (done) -> - RealTimeClient.setSession({}, done) - return null + return describe('with a valid cookie and a matching session', function() { + before(function(done) { + RealTimeClient.setSession({}, done); + return null; + }); - it 'should not return an error', (done) -> - @checkSocket (error) -> - expect(error).to.not.exist - done() + return it('should not return an error', function(done) { + return this.checkSocket(function(error) { + expect(error).to.not.exist; + return done(); + }); + }); + }); +}); diff --git a/services/real-time/test/acceptance/coffee/SessionTests.js b/services/real-time/test/acceptance/coffee/SessionTests.js index 23c4e78ce9..b8da531875 100644 --- a/services/real-time/test/acceptance/coffee/SessionTests.js +++ b/services/real-time/test/acceptance/coffee/SessionTests.js @@ -1,35 +1,51 @@ -chai = require("chai") -expect = chai.expect +/* + * decaffeinate suggestions: + * DS101: Remove unnecessary use of Array.from + * DS102: Remove unnecessary code created because of implicit returns + * DS207: Consider shorter variations of null checks + * Full docs: https://github.com/decaffeinate/decaffeinate/blob/master/docs/suggestions.md + */ +const chai = require("chai"); +const { + expect +} = chai; -RealTimeClient = require "./helpers/RealTimeClient" +const RealTimeClient = require("./helpers/RealTimeClient"); -describe "Session", -> - describe "with an established session", -> - before (done) -> - @user_id = "mock-user-id" - RealTimeClient.setSession { - user: { _id: @user_id } - }, (error) => - throw error if error? - @client = RealTimeClient.connect() - return done() - return null +describe("Session", () => describe("with an established session", function() { + before(function(done) { + this.user_id = "mock-user-id"; + RealTimeClient.setSession({ + user: { _id: this.user_id } + }, error => { + if (error != null) { throw error; } + this.client = RealTimeClient.connect(); + return done(); + }); + return null; + }); - it "should not get disconnected", (done) -> - disconnected = false - @client.on "disconnect", () -> - disconnected = true - setTimeout () => - expect(disconnected).to.equal false - done() - , 500 - - it "should appear in the list of connected clients", (done) -> - RealTimeClient.getConnectedClients (error, clients) => - included = false - for client in clients - if client.client_id == @client.socket.sessionid - included = true - break - expect(included).to.equal true - done() + it("should not get disconnected", function(done) { + let disconnected = false; + this.client.on("disconnect", () => disconnected = true); + return setTimeout(() => { + expect(disconnected).to.equal(false); + return done(); + } + , 500); + }); + + return it("should appear in the list of connected clients", function(done) { + return RealTimeClient.getConnectedClients((error, clients) => { + let included = false; + for (let client of Array.from(clients)) { + if (client.client_id === this.client.socket.sessionid) { + included = true; + break; + } + } + expect(included).to.equal(true); + return done(); + }); + }); +})); diff --git a/services/real-time/test/acceptance/coffee/helpers/FixturesManager.js b/services/real-time/test/acceptance/coffee/helpers/FixturesManager.js index 0889b45c2a..bdae3e01e1 100644 --- a/services/real-time/test/acceptance/coffee/helpers/FixturesManager.js +++ b/services/real-time/test/acceptance/coffee/helpers/FixturesManager.js @@ -1,48 +1,67 @@ -RealTimeClient = require "./RealTimeClient" -MockWebServer = require "./MockWebServer" -MockDocUpdaterServer = require "./MockDocUpdaterServer" +/* + * decaffeinate suggestions: + * DS102: Remove unnecessary code created because of implicit returns + * DS207: Consider shorter variations of null checks + * Full docs: https://github.com/decaffeinate/decaffeinate/blob/master/docs/suggestions.md + */ +let FixturesManager; +const RealTimeClient = require("./RealTimeClient"); +const MockWebServer = require("./MockWebServer"); +const MockDocUpdaterServer = require("./MockDocUpdaterServer"); -module.exports = FixturesManager = - setUpProject: (options = {}, callback = (error, data) ->) -> - options.user_id ||= FixturesManager.getRandomId() - options.project_id ||= FixturesManager.getRandomId() - options.project ||= { name: "Test Project" } - {project_id, user_id, privilegeLevel, project, publicAccess} = options +module.exports = (FixturesManager = { + setUpProject(options, callback) { + if (options == null) { options = {}; } + if (callback == null) { callback = function(error, data) {}; } + if (!options.user_id) { options.user_id = FixturesManager.getRandomId(); } + if (!options.project_id) { options.project_id = FixturesManager.getRandomId(); } + if (!options.project) { options.project = { name: "Test Project" }; } + const {project_id, user_id, privilegeLevel, project, publicAccess} = options; - privileges = {} - privileges[user_id] = privilegeLevel - if publicAccess - privileges["anonymous-user"] = publicAccess + const privileges = {}; + privileges[user_id] = privilegeLevel; + if (publicAccess) { + privileges["anonymous-user"] = publicAccess; + } - MockWebServer.createMockProject(project_id, privileges, project) - MockWebServer.run (error) => - throw error if error? - RealTimeClient.setSession { + MockWebServer.createMockProject(project_id, privileges, project); + return MockWebServer.run(error => { + if (error != null) { throw error; } + return RealTimeClient.setSession({ user: { - _id: user_id - first_name: "Joe" + _id: user_id, + first_name: "Joe", last_name: "Bloggs" } - }, (error) => - throw error if error? - callback null, {project_id, user_id, privilegeLevel, project} + }, error => { + if (error != null) { throw error; } + return callback(null, {project_id, user_id, privilegeLevel, project}); + }); + }); + }, - setUpDoc: (project_id, options = {}, callback = (error, data) ->) -> - options.doc_id ||= FixturesManager.getRandomId() - options.lines ||= ["doc", "lines"] - options.version ||= 42 - options.ops ||= ["mock", "ops"] - {doc_id, lines, version, ops, ranges} = options + setUpDoc(project_id, options, callback) { + if (options == null) { options = {}; } + if (callback == null) { callback = function(error, data) {}; } + if (!options.doc_id) { options.doc_id = FixturesManager.getRandomId(); } + if (!options.lines) { options.lines = ["doc", "lines"]; } + if (!options.version) { options.version = 42; } + if (!options.ops) { options.ops = ["mock", "ops"]; } + const {doc_id, lines, version, ops, ranges} = options; - MockDocUpdaterServer.createMockDoc project_id, doc_id, {lines, version, ops, ranges} - MockDocUpdaterServer.run (error) => - throw error if error? - callback null, {project_id, doc_id, lines, version, ops} + MockDocUpdaterServer.createMockDoc(project_id, doc_id, {lines, version, ops, ranges}); + return MockDocUpdaterServer.run(error => { + if (error != null) { throw error; } + return callback(null, {project_id, doc_id, lines, version, ops}); + }); + }, - getRandomId: () -> + getRandomId() { return require("crypto") .createHash("sha1") .update(Math.random().toString()) .digest("hex") - .slice(0,24) + .slice(0,24); + } +}); \ No newline at end of file diff --git a/services/real-time/test/acceptance/coffee/helpers/MockDocUpdaterServer.js b/services/real-time/test/acceptance/coffee/helpers/MockDocUpdaterServer.js index ac5bfc7093..675c07a0ed 100644 --- a/services/real-time/test/acceptance/coffee/helpers/MockDocUpdaterServer.js +++ b/services/real-time/test/acceptance/coffee/helpers/MockDocUpdaterServer.js @@ -1,46 +1,65 @@ -sinon = require "sinon" -express = require "express" +/* + * decaffeinate suggestions: + * DS102: Remove unnecessary code created because of implicit returns + * DS207: Consider shorter variations of null checks + * Full docs: https://github.com/decaffeinate/decaffeinate/blob/master/docs/suggestions.md + */ +let MockDocUpdaterServer; +const sinon = require("sinon"); +const express = require("express"); -module.exports = MockDocUpdaterServer = - docs: {} +module.exports = (MockDocUpdaterServer = { + docs: {}, - createMockDoc: (project_id, doc_id, data) -> - MockDocUpdaterServer.docs["#{project_id}:#{doc_id}"] = data + createMockDoc(project_id, doc_id, data) { + return MockDocUpdaterServer.docs[`${project_id}:${doc_id}`] = data; + }, - getDocument: (project_id, doc_id, fromVersion, callback = (error, data) ->) -> - callback( - null, MockDocUpdaterServer.docs["#{project_id}:#{doc_id}"] - ) + getDocument(project_id, doc_id, fromVersion, callback) { + if (callback == null) { callback = function(error, data) {}; } + return callback( + null, MockDocUpdaterServer.docs[`${project_id}:${doc_id}`] + ); + }, - deleteProject: sinon.stub().callsArg(1) + deleteProject: sinon.stub().callsArg(1), - getDocumentRequest: (req, res, next) -> - {project_id, doc_id} = req.params - {fromVersion} = req.query - fromVersion = parseInt(fromVersion, 10) - MockDocUpdaterServer.getDocument project_id, doc_id, fromVersion, (error, data) -> - return next(error) if error? - res.json data + getDocumentRequest(req, res, next) { + const {project_id, doc_id} = req.params; + let {fromVersion} = req.query; + fromVersion = parseInt(fromVersion, 10); + return MockDocUpdaterServer.getDocument(project_id, doc_id, fromVersion, function(error, data) { + if (error != null) { return next(error); } + return res.json(data); + }); + }, - deleteProjectRequest: (req, res, next) -> - {project_id} = req.params - MockDocUpdaterServer.deleteProject project_id, (error) -> - return next(error) if error? - res.sendStatus 204 + deleteProjectRequest(req, res, next) { + const {project_id} = req.params; + return MockDocUpdaterServer.deleteProject(project_id, function(error) { + if (error != null) { return next(error); } + return res.sendStatus(204); + }); + }, - running: false - run: (callback = (error) ->) -> - if MockDocUpdaterServer.running - return callback() - app = express() - app.get "/project/:project_id/doc/:doc_id", MockDocUpdaterServer.getDocumentRequest - app.delete "/project/:project_id", MockDocUpdaterServer.deleteProjectRequest - app.listen 3003, (error) -> - MockDocUpdaterServer.running = true - callback(error) - .on "error", (error) -> - console.error "error starting MockDocUpdaterServer:", error.message - process.exit(1) + running: false, + run(callback) { + if (callback == null) { callback = function(error) {}; } + if (MockDocUpdaterServer.running) { + return callback(); + } + const app = express(); + app.get("/project/:project_id/doc/:doc_id", MockDocUpdaterServer.getDocumentRequest); + app.delete("/project/:project_id", MockDocUpdaterServer.deleteProjectRequest); + return app.listen(3003, function(error) { + MockDocUpdaterServer.running = true; + return callback(error); + }).on("error", function(error) { + console.error("error starting MockDocUpdaterServer:", error.message); + return process.exit(1); + }); + } +}); -sinon.spy MockDocUpdaterServer, "getDocument" +sinon.spy(MockDocUpdaterServer, "getDocument"); diff --git a/services/real-time/test/acceptance/coffee/helpers/MockWebServer.js b/services/real-time/test/acceptance/coffee/helpers/MockWebServer.js index 7c479c59bb..3fb8db33e7 100644 --- a/services/real-time/test/acceptance/coffee/helpers/MockWebServer.js +++ b/services/real-time/test/acceptance/coffee/helpers/MockWebServer.js @@ -1,46 +1,64 @@ -sinon = require "sinon" -express = require "express" +/* + * decaffeinate suggestions: + * DS102: Remove unnecessary code created because of implicit returns + * DS207: Consider shorter variations of null checks + * Full docs: https://github.com/decaffeinate/decaffeinate/blob/master/docs/suggestions.md + */ +let MockWebServer; +const sinon = require("sinon"); +const express = require("express"); -module.exports = MockWebServer = - projects: {} - privileges: {} +module.exports = (MockWebServer = { + projects: {}, + privileges: {}, - createMockProject: (project_id, privileges, project) -> - MockWebServer.privileges[project_id] = privileges - MockWebServer.projects[project_id] = project + createMockProject(project_id, privileges, project) { + MockWebServer.privileges[project_id] = privileges; + return MockWebServer.projects[project_id] = project; + }, - joinProject: (project_id, user_id, callback = (error, project, privilegeLevel) ->) -> - callback( + joinProject(project_id, user_id, callback) { + if (callback == null) { callback = function(error, project, privilegeLevel) {}; } + return callback( null, MockWebServer.projects[project_id], MockWebServer.privileges[project_id][user_id] - ) + ); + }, - joinProjectRequest: (req, res, next) -> - {project_id} = req.params - {user_id} = req.query - if project_id == 'rate-limited' - res.status(429).send() - else - MockWebServer.joinProject project_id, user_id, (error, project, privilegeLevel) -> - return next(error) if error? - res.json { - project: project - privilegeLevel: privilegeLevel - } + joinProjectRequest(req, res, next) { + const {project_id} = req.params; + const {user_id} = req.query; + if (project_id === 'rate-limited') { + return res.status(429).send(); + } else { + return MockWebServer.joinProject(project_id, user_id, function(error, project, privilegeLevel) { + if (error != null) { return next(error); } + return res.json({ + project, + privilegeLevel + }); + }); + } + }, - running: false - run: (callback = (error) ->) -> - if MockWebServer.running - return callback() - app = express() - app.post "/project/:project_id/join", MockWebServer.joinProjectRequest - app.listen 3000, (error) -> - MockWebServer.running = true - callback(error) - .on "error", (error) -> - console.error "error starting MockWebServer:", error.message - process.exit(1) + running: false, + run(callback) { + if (callback == null) { callback = function(error) {}; } + if (MockWebServer.running) { + return callback(); + } + const app = express(); + app.post("/project/:project_id/join", MockWebServer.joinProjectRequest); + return app.listen(3000, function(error) { + MockWebServer.running = true; + return callback(error); + }).on("error", function(error) { + console.error("error starting MockWebServer:", error.message); + return process.exit(1); + }); + } +}); -sinon.spy MockWebServer, "joinProject" +sinon.spy(MockWebServer, "joinProject"); diff --git a/services/real-time/test/acceptance/coffee/helpers/RealTimeClient.js b/services/real-time/test/acceptance/coffee/helpers/RealTimeClient.js index 7d54e23b3c..5fc8466a2a 100644 --- a/services/real-time/test/acceptance/coffee/helpers/RealTimeClient.js +++ b/services/real-time/test/acceptance/coffee/helpers/RealTimeClient.js @@ -1,75 +1,94 @@ -XMLHttpRequest = require("../../libs/XMLHttpRequest").XMLHttpRequest -io = require("socket.io-client") -async = require("async") +/* + * decaffeinate suggestions: + * DS102: Remove unnecessary code created because of implicit returns + * DS207: Consider shorter variations of null checks + * Full docs: https://github.com/decaffeinate/decaffeinate/blob/master/docs/suggestions.md + */ +let Client; +const { + XMLHttpRequest +} = require("../../libs/XMLHttpRequest"); +const io = require("socket.io-client"); +const async = require("async"); -request = require "request" -Settings = require "settings-sharelatex" -redis = require "redis-sharelatex" -rclient = redis.createClient(Settings.redis.websessions) +const request = require("request"); +const Settings = require("settings-sharelatex"); +const redis = require("redis-sharelatex"); +const rclient = redis.createClient(Settings.redis.websessions); -uid = require('uid-safe').sync -signature = require("cookie-signature") +const uid = require('uid-safe').sync; +const signature = require("cookie-signature"); -io.util.request = () -> - xhr = new XMLHttpRequest() - _open = xhr.open - xhr.open = () => - _open.apply(xhr, arguments) - if Client.cookie? - xhr.setRequestHeader("Cookie", Client.cookie) - return xhr +io.util.request = function() { + const xhr = new XMLHttpRequest(); + const _open = xhr.open; + xhr.open = function() { + _open.apply(xhr, arguments); + if (Client.cookie != null) { + return xhr.setRequestHeader("Cookie", Client.cookie); + } + }.bind(this); + return xhr; +}; -module.exports = Client = - cookie: null +module.exports = (Client = { + cookie: null, - setSession: (session, callback = (error) ->) -> - sessionId = uid(24) - session.cookie = {} - rclient.set "sess:" + sessionId, JSON.stringify(session), (error) -> - return callback(error) if error? - secret = Settings.security.sessionSecret - cookieKey = 's:' + signature.sign(sessionId, secret) - Client.cookie = "#{Settings.cookieName}=#{cookieKey}" - callback() + setSession(session, callback) { + if (callback == null) { callback = function(error) {}; } + const sessionId = uid(24); + session.cookie = {}; + return rclient.set("sess:" + sessionId, JSON.stringify(session), function(error) { + if (error != null) { return callback(error); } + const secret = Settings.security.sessionSecret; + const cookieKey = 's:' + signature.sign(sessionId, secret); + Client.cookie = `${Settings.cookieName}=${cookieKey}`; + return callback(); + }); + }, - unsetSession: (callback = (error) ->) -> - Client.cookie = null - callback() + unsetSession(callback) { + if (callback == null) { callback = function(error) {}; } + Client.cookie = null; + return callback(); + }, - connect: (cookie) -> - client = io.connect("http://localhost:3026", 'force new connection': true) - client.on 'connectionAccepted', (_, publicId) -> - client.publicId = publicId - return client + connect(cookie) { + const client = io.connect("http://localhost:3026", {'force new connection': true}); + client.on('connectionAccepted', (_, publicId) => client.publicId = publicId); + return client; + }, - getConnectedClients: (callback = (error, clients) ->) -> - request.get { - url: "http://localhost:3026/clients" + getConnectedClients(callback) { + if (callback == null) { callback = function(error, clients) {}; } + return request.get({ + url: "http://localhost:3026/clients", json: true - }, (error, response, data) -> - callback error, data + }, (error, response, data) => callback(error, data)); + }, - getConnectedClient: (client_id, callback = (error, clients) ->) -> - request.get { - url: "http://localhost:3026/clients/#{client_id}" + getConnectedClient(client_id, callback) { + if (callback == null) { callback = function(error, clients) {}; } + return request.get({ + url: `http://localhost:3026/clients/${client_id}`, json: true - }, (error, response, data) -> - callback error, data + }, (error, response, data) => callback(error, data)); + }, - disconnectClient: (client_id, callback) -> - request.post { - url: "http://localhost:3026/client/#{client_id}/disconnect" + disconnectClient(client_id, callback) { + request.post({ + url: `http://localhost:3026/client/${client_id}/disconnect`, auth: { user: Settings.internal.realTime.user, pass: Settings.internal.realTime.pass } - }, (error, response, data) -> - callback error, data - return null + }, (error, response, data) => callback(error, data)); + return null; + }, - disconnectAllClients: (callback) -> - Client.getConnectedClients (error, clients) -> - async.each clients, (clientView, cb) -> - Client.disconnectClient clientView.client_id, cb - , callback + disconnectAllClients(callback) { + return Client.getConnectedClients((error, clients) => async.each(clients, (clientView, cb) => Client.disconnectClient(clientView.client_id, cb) + , callback)); + } +}); diff --git a/services/real-time/test/acceptance/coffee/helpers/RealtimeServer.js b/services/real-time/test/acceptance/coffee/helpers/RealtimeServer.js index 3a721c18ed..7c97b003ef 100644 --- a/services/real-time/test/acceptance/coffee/helpers/RealtimeServer.js +++ b/services/real-time/test/acceptance/coffee/helpers/RealtimeServer.js @@ -1,23 +1,46 @@ -app = require('../../../../app') -logger = require("logger-sharelatex") -Settings = require("settings-sharelatex") +/* + * decaffeinate suggestions: + * DS101: Remove unnecessary use of Array.from + * DS102: Remove unnecessary code created because of implicit returns + * DS103: Rewrite code to no longer use __guard__ + * DS205: Consider reworking code to avoid use of IIFEs + * DS207: Consider shorter variations of null checks + * Full docs: https://github.com/decaffeinate/decaffeinate/blob/master/docs/suggestions.md + */ +const app = require('../../../../app'); +const logger = require("logger-sharelatex"); +const Settings = require("settings-sharelatex"); -module.exports = - running: false - initing: false - callbacks: [] - ensureRunning: (callback = (error) ->) -> - if @running - return callback() - else if @initing - @callbacks.push callback - else - @initing = true - @callbacks.push callback - app.listen Settings.internal?.realtime?.port, "localhost", (error) => - throw error if error? - @running = true - logger.log("clsi running in dev mode") +module.exports = { + running: false, + initing: false, + callbacks: [], + ensureRunning(callback) { + if (callback == null) { callback = function(error) {}; } + if (this.running) { + return callback(); + } else if (this.initing) { + return this.callbacks.push(callback); + } else { + this.initing = true; + this.callbacks.push(callback); + return app.listen(__guard__(Settings.internal != null ? Settings.internal.realtime : undefined, x => x.port), "localhost", error => { + if (error != null) { throw error; } + this.running = true; + logger.log("clsi running in dev mode"); - for callback in @callbacks - callback() + return (() => { + const result = []; + for (callback of Array.from(this.callbacks)) { + result.push(callback()); + } + return result; + })(); + }); + } + } +}; + +function __guard__(value, transform) { + return (typeof value !== 'undefined' && value !== null) ? transform(value) : undefined; +} \ No newline at end of file From 5443450abb22b7958aff907c8016057915675a4a Mon Sep 17 00:00:00 2001 From: decaffeinate Date: Tue, 23 Jun 2020 18:30:34 +0100 Subject: [PATCH 380/491] decaffeinate: Run post-processing cleanups on ApplyUpdateTests.coffee and 18 other files --- .../acceptance/coffee/ApplyUpdateTests.js | 7 +++ .../acceptance/coffee/ClientTrackingTests.js | 13 ++++-- .../acceptance/coffee/DrainManagerTests.js | 13 ++++-- .../test/acceptance/coffee/EarlyDisconnect.js | 16 ++++--- .../acceptance/coffee/HttpControllerTests.js | 11 +++-- .../test/acceptance/coffee/JoinDocTests.js | 7 +++ .../acceptance/coffee/JoinProjectTests.js | 8 +++- .../test/acceptance/coffee/LeaveDocTests.js | 22 +++++++--- .../acceptance/coffee/LeaveProjectTests.js | 11 ++++- .../test/acceptance/coffee/PubSubRace.js | 44 +++++++++++-------- .../acceptance/coffee/ReceiveUpdateTests.js | 7 +++ .../test/acceptance/coffee/RouterTests.js | 11 +++-- .../acceptance/coffee/SessionSocketsTests.js | 23 ++++++---- .../test/acceptance/coffee/SessionTests.js | 12 +++-- .../coffee/helpers/FixturesManager.js | 6 +++ .../coffee/helpers/MockDocUpdaterServer.js | 15 +++++-- .../coffee/helpers/MockWebServer.js | 13 ++++-- .../coffee/helpers/RealTimeClient.js | 11 ++++- .../coffee/helpers/RealtimeServer.js | 5 +++ 19 files changed, 187 insertions(+), 68 deletions(-) diff --git a/services/real-time/test/acceptance/coffee/ApplyUpdateTests.js b/services/real-time/test/acceptance/coffee/ApplyUpdateTests.js index 41d1a5e100..bc9d315b0f 100644 --- a/services/real-time/test/acceptance/coffee/ApplyUpdateTests.js +++ b/services/real-time/test/acceptance/coffee/ApplyUpdateTests.js @@ -1,3 +1,10 @@ +/* eslint-disable + camelcase, + handle-callback-err, + no-return-assign, +*/ +// TODO: This file was created by bulk-decaffeinate. +// Fix any style issues and re-enable lint. /* * decaffeinate suggestions: * DS101: Remove unnecessary use of Array.from diff --git a/services/real-time/test/acceptance/coffee/ClientTrackingTests.js b/services/real-time/test/acceptance/coffee/ClientTrackingTests.js index 8406aebcbe..75e23d0719 100644 --- a/services/real-time/test/acceptance/coffee/ClientTrackingTests.js +++ b/services/real-time/test/acceptance/coffee/ClientTrackingTests.js @@ -1,3 +1,10 @@ +/* eslint-disable + camelcase, + handle-callback-err, + no-unused-vars, +*/ +// TODO: This file was created by bulk-decaffeinate. +// Fix any style issues and re-enable lint. /* * decaffeinate suggestions: * DS101: Remove unnecessary use of Array.from @@ -71,7 +78,7 @@ describe("clientTracking", function() { row: (this.row = 42), column: (this.column = 36), doc_id: this.doc_id - }, function(error) { + }, (error) => { if (error != null) { throw error; } return setTimeout(cb, 300); }); @@ -94,7 +101,7 @@ describe("clientTracking", function() { return it("should record the update in getConnectedUsers", function(done) { return this.clientB.emit("clientTracking.getConnectedUsers", (error, users) => { - for (let user of Array.from(users)) { + for (const user of Array.from(users)) { if (user.client_id === this.clientA.publicId) { expect(user.cursorData).to.deep.equal({ row: this.row, @@ -167,7 +174,7 @@ describe("clientTracking", function() { row: (this.row = 42), column: (this.column = 36), doc_id: this.doc_id - }, function(error) { + }, (error) => { if (error != null) { throw error; } return setTimeout(cb, 300); }); diff --git a/services/real-time/test/acceptance/coffee/DrainManagerTests.js b/services/real-time/test/acceptance/coffee/DrainManagerTests.js index 91bf8836ec..1ff4a4afbb 100644 --- a/services/real-time/test/acceptance/coffee/DrainManagerTests.js +++ b/services/real-time/test/acceptance/coffee/DrainManagerTests.js @@ -1,3 +1,8 @@ +/* eslint-disable + camelcase, +*/ +// TODO: This file was created by bulk-decaffeinate. +// Fix any style issues and re-enable lint. /* * decaffeinate suggestions: * DS102: Remove unnecessary code created because of implicit returns @@ -44,11 +49,11 @@ describe("DrainManagerTests", function() { }); // trigger and check cleanup - it("should have disconnected all previous clients", done => RealTimeClient.getConnectedClients(function(error, data) { + it("should have disconnected all previous clients", function(done) { return RealTimeClient.getConnectedClients((error, data) => { if (error) { return done(error); } expect(data.length).to.equal(0); return done(); - })); + }); }); return describe("with two clients in the project", function() { beforeEach(function(done) { @@ -87,9 +92,9 @@ describe("DrainManagerTests", function() { ], done); }); - afterEach(done => drain(0, done)); // reset drain + afterEach(function(done) { return drain(0, done); }); // reset drain - it("should not timeout", () => expect(true).to.equal(true)); + it("should not timeout", function() { return expect(true).to.equal(true); }); return it("should not have disconnected", function() { expect(this.clientA.socket.connected).to.equal(true); diff --git a/services/real-time/test/acceptance/coffee/EarlyDisconnect.js b/services/real-time/test/acceptance/coffee/EarlyDisconnect.js index 875f33ee33..427d77040b 100644 --- a/services/real-time/test/acceptance/coffee/EarlyDisconnect.js +++ b/services/real-time/test/acceptance/coffee/EarlyDisconnect.js @@ -1,3 +1,9 @@ +/* eslint-disable + camelcase, + no-return-assign, +*/ +// TODO: This file was created by bulk-decaffeinate. +// Fix any style issues and re-enable lint. /* * decaffeinate suggestions: * DS101: Remove unnecessary use of Array.from @@ -19,7 +25,7 @@ const rclientRT = redis.createClient(settings.redis.realtime); const KeysRT = settings.redis.realtime.key_schema; describe("EarlyDisconnect", function() { - before(done => MockDocUpdaterServer.run(done)); + before(function(done) { return MockDocUpdaterServer.run(done); }); describe("when the client disconnects before joinProject completes", function() { before(function() { @@ -51,7 +57,7 @@ describe("EarlyDisconnect", function() { }, cb => { - this.clientA.emit("joinProject", {project_id: this.project_id}, (function() {})); + this.clientA.emit("joinProject", {project_id: this.project_id}, (() => {})); // disconnect before joinProject completes this.clientA.on("disconnect", () => cb()); return this.clientA.disconnect(); @@ -110,7 +116,7 @@ describe("EarlyDisconnect", function() { }, cb => { - this.clientA.emit("joinDoc", this.doc_id, (function() {})); + this.clientA.emit("joinDoc", this.doc_id, (() => {})); // disconnect before joinDoc completes this.clientA.on("disconnect", () => cb()); return this.clientA.disconnect(); @@ -182,7 +188,7 @@ describe("EarlyDisconnect", function() { row: 42, column: 36, doc_id: this.doc_id - }, (function() {})); + }, (() => {})); // disconnect before updateClientPosition completes this.clientA.on("disconnect", () => cb()); return this.clientA.disconnect(); @@ -198,7 +204,7 @@ describe("EarlyDisconnect", function() { // we can not force the race condition, so we have to try many times return Array.from(Array.from({length: 20}).map((_, i) => i+1)).map((attempt) => it(`should not show the client as connected (race ${attempt})`, function(done) { - rclientRT.smembers(KeysRT.clientsInProject({project_id: this.project_id}), function(err, results) { + rclientRT.smembers(KeysRT.clientsInProject({project_id: this.project_id}), (err, results) => { if (err) { return done(err); } expect(results).to.deep.equal([]); return done(); diff --git a/services/real-time/test/acceptance/coffee/HttpControllerTests.js b/services/real-time/test/acceptance/coffee/HttpControllerTests.js index 701b1f7d23..c701a91e20 100644 --- a/services/real-time/test/acceptance/coffee/HttpControllerTests.js +++ b/services/real-time/test/acceptance/coffee/HttpControllerTests.js @@ -1,3 +1,8 @@ +/* eslint-disable + camelcase, +*/ +// TODO: This file was created by bulk-decaffeinate. +// Fix any style issues and re-enable lint. /* * decaffeinate suggestions: * DS102: Remove unnecessary code created because of implicit returns @@ -15,17 +20,17 @@ const RealTimeClient = require("./helpers/RealTimeClient"); const FixturesManager = require("./helpers/FixturesManager"); describe('HttpControllerTests', function() { - describe('without a user', () => it('should return 404 for the client view', function(done) { + describe('without a user', function() { return it('should return 404 for the client view', function(done) { const client_id = 'not-existing'; return request.get({ url: `/clients/${client_id}`, json: true - }, function(error, response, data) { + }, (error, response, data) => { if (error) { return done(error); } expect(response.statusCode).to.equal(404); return done(); }); - })); + }); }); return describe('with a user and after joining a project', function() { before(function(done) { diff --git a/services/real-time/test/acceptance/coffee/JoinDocTests.js b/services/real-time/test/acceptance/coffee/JoinDocTests.js index 0026a6e858..f55af76820 100644 --- a/services/real-time/test/acceptance/coffee/JoinDocTests.js +++ b/services/real-time/test/acceptance/coffee/JoinDocTests.js @@ -1,3 +1,10 @@ +/* eslint-disable + camelcase, + handle-callback-err, + no-return-assign, +*/ +// TODO: This file was created by bulk-decaffeinate. +// Fix any style issues and re-enable lint. /* * decaffeinate suggestions: * DS101: Remove unnecessary use of Array.from diff --git a/services/real-time/test/acceptance/coffee/JoinProjectTests.js b/services/real-time/test/acceptance/coffee/JoinProjectTests.js index 3f962e2c12..f84af3bba5 100644 --- a/services/real-time/test/acceptance/coffee/JoinProjectTests.js +++ b/services/real-time/test/acceptance/coffee/JoinProjectTests.js @@ -1,3 +1,9 @@ +/* eslint-disable + camelcase, + handle-callback-err, +*/ +// TODO: This file was created by bulk-decaffeinate. +// Fix any style issues and re-enable lint. /* * decaffeinate suggestions: * DS101: Remove unnecessary use of Array.from @@ -79,7 +85,7 @@ describe("joinProject", function() { return it("should have marked the user as connected", function(done) { return this.client.emit("clientTracking.getConnectedUsers", (error, users) => { let connected = false; - for (let user of Array.from(users)) { + for (const user of Array.from(users)) { if ((user.client_id === this.client.publicId) && (user.user_id === this.user_id)) { connected = true; break; diff --git a/services/real-time/test/acceptance/coffee/LeaveDocTests.js b/services/real-time/test/acceptance/coffee/LeaveDocTests.js index 5e589356f9..3f396e3df5 100644 --- a/services/real-time/test/acceptance/coffee/LeaveDocTests.js +++ b/services/real-time/test/acceptance/coffee/LeaveDocTests.js @@ -1,3 +1,11 @@ +/* eslint-disable + camelcase, + handle-callback-err, + no-return-assign, + no-unused-vars, +*/ +// TODO: This file was created by bulk-decaffeinate. +// Fix any style issues and re-enable lint. /* * decaffeinate suggestions: * DS101: Remove unnecessary use of Array.from @@ -73,7 +81,7 @@ describe("leaveDoc", function() { describe("then leaving the doc", function() { beforeEach(function(done) { - return this.client.emit("leaveDoc", this.doc_id, function(error) { + return this.client.emit("leaveDoc", this.doc_id, (error) => { if (error != null) { throw error; } return done(); }); @@ -89,15 +97,15 @@ describe("leaveDoc", function() { describe("when sending a leaveDoc request before the previous joinDoc request has completed", function() { beforeEach(function(done) { - this.client.emit("leaveDoc", this.doc_id, function() {}); - this.client.emit("joinDoc", this.doc_id, function() {}); - return this.client.emit("leaveDoc", this.doc_id, function(error) { + this.client.emit("leaveDoc", this.doc_id, () => {}); + this.client.emit("joinDoc", this.doc_id, () => {}); + return this.client.emit("leaveDoc", this.doc_id, (error) => { if (error != null) { throw error; } return done(); }); }); - it("should not trigger an error", () => sinon.assert.neverCalledWith(logger.error, sinon.match.any, "not subscribed - shouldn't happen")); + it("should not trigger an error", function() { return sinon.assert.neverCalledWith(logger.error, sinon.match.any, "not subscribed - shouldn't happen"); }); return it("should have left the doc room", function(done) { return RealTimeClient.getConnectedClient(this.client.socket.sessionid, (error, client) => { @@ -109,13 +117,13 @@ describe("leaveDoc", function() { return describe("when sending a leaveDoc for a room the client has not joined ", function() { beforeEach(function(done) { - return this.client.emit("leaveDoc", this.other_doc_id, function(error) { + return this.client.emit("leaveDoc", this.other_doc_id, (error) => { if (error != null) { throw error; } return done(); }); }); - return it("should trigger a low level message only", () => sinon.assert.calledWith(logger.log, sinon.match.any, "ignoring request from client to leave room it is not in")); + return it("should trigger a low level message only", function() { return sinon.assert.calledWith(logger.log, sinon.match.any, "ignoring request from client to leave room it is not in"); }); }); }); }); diff --git a/services/real-time/test/acceptance/coffee/LeaveProjectTests.js b/services/real-time/test/acceptance/coffee/LeaveProjectTests.js index 11e4ed2471..36a17fe081 100644 --- a/services/real-time/test/acceptance/coffee/LeaveProjectTests.js +++ b/services/real-time/test/acceptance/coffee/LeaveProjectTests.js @@ -1,3 +1,10 @@ +/* eslint-disable + camelcase, + handle-callback-err, + no-throw-literal, +*/ +// TODO: This file was created by bulk-decaffeinate. +// Fix any style issues and re-enable lint. /* * decaffeinate suggestions: * DS101: Remove unnecessary use of Array.from @@ -15,7 +22,7 @@ const redis = require("redis-sharelatex"); const rclient = redis.createClient(settings.redis.pubsub); describe("leaveProject", function() { - before(done => MockDocUpdaterServer.run(done)); + before(function(done) { return MockDocUpdaterServer.run(done); }); describe("with other clients in the project", function() { before(function(done) { @@ -96,7 +103,7 @@ describe("leaveProject", function() { it("should no longer list the client in connected users", function(done) { return this.clientB.emit("clientTracking.getConnectedUsers", (error, users) => { - for (let user of Array.from(users)) { + for (const user of Array.from(users)) { if (user.client_id === this.clientA.publicId) { throw "Expected clientA to not be listed in connected users"; } diff --git a/services/real-time/test/acceptance/coffee/PubSubRace.js b/services/real-time/test/acceptance/coffee/PubSubRace.js index 3c5a6f0669..34d526d820 100644 --- a/services/real-time/test/acceptance/coffee/PubSubRace.js +++ b/services/real-time/test/acceptance/coffee/PubSubRace.js @@ -1,3 +1,9 @@ +/* eslint-disable + camelcase, + no-return-assign, +*/ +// TODO: This file was created by bulk-decaffeinate. +// Fix any style issues and re-enable lint. /* * decaffeinate suggestions: * DS102: Remove unnecessary code created because of implicit returns @@ -14,7 +20,7 @@ const redis = require("redis-sharelatex"); const rclient = redis.createClient(settings.redis.pubsub); describe("PubSubRace", function() { - before(done => MockDocUpdaterServer.run(done)); + before(function(done) { return MockDocUpdaterServer.run(done); }); describe("when the client leaves a doc before joinDoc completes", function() { before(function(done) { @@ -50,7 +56,7 @@ describe("PubSubRace", function() { }, cb => { - this.clientA.emit("joinDoc", this.doc_id, function() {}); + this.clientA.emit("joinDoc", this.doc_id, () => {}); // leave before joinDoc completes return this.clientA.emit("leaveDoc", this.doc_id, cb); }, @@ -106,15 +112,15 @@ describe("PubSubRace", function() { }, cb => { - this.clientA.emit("joinDoc", this.doc_id, function() {}); - this.clientA.emit("leaveDoc", this.doc_id, function() {}); - this.clientA.emit("joinDoc", this.doc_id, function() {}); - this.clientA.emit("leaveDoc", this.doc_id, function() {}); - this.clientA.emit("joinDoc", this.doc_id, function() {}); - this.clientA.emit("leaveDoc", this.doc_id, function() {}); - this.clientA.emit("joinDoc", this.doc_id, function() {}); - this.clientA.emit("leaveDoc", this.doc_id, function() {}); - this.clientA.emit("joinDoc", this.doc_id, function() {}); + this.clientA.emit("joinDoc", this.doc_id, () => {}); + this.clientA.emit("leaveDoc", this.doc_id, () => {}); + this.clientA.emit("joinDoc", this.doc_id, () => {}); + this.clientA.emit("leaveDoc", this.doc_id, () => {}); + this.clientA.emit("joinDoc", this.doc_id, () => {}); + this.clientA.emit("leaveDoc", this.doc_id, () => {}); + this.clientA.emit("joinDoc", this.doc_id, () => {}); + this.clientA.emit("leaveDoc", this.doc_id, () => {}); + this.clientA.emit("joinDoc", this.doc_id, () => {}); return this.clientA.emit("leaveDoc", this.doc_id, cb); }, @@ -169,14 +175,14 @@ describe("PubSubRace", function() { }, cb => { - this.clientA.emit("joinDoc", this.doc_id, function() {}); - this.clientA.emit("leaveDoc", this.doc_id, function() {}); - this.clientA.emit("joinDoc", this.doc_id, function() {}); - this.clientA.emit("leaveDoc", this.doc_id, function() {}); - this.clientA.emit("joinDoc", this.doc_id, function() {}); - this.clientA.emit("leaveDoc", this.doc_id, function() {}); - this.clientA.emit("joinDoc", this.doc_id, function() {}); - this.clientA.emit("leaveDoc", this.doc_id, function() {}); + this.clientA.emit("joinDoc", this.doc_id, () => {}); + this.clientA.emit("leaveDoc", this.doc_id, () => {}); + this.clientA.emit("joinDoc", this.doc_id, () => {}); + this.clientA.emit("leaveDoc", this.doc_id, () => {}); + this.clientA.emit("joinDoc", this.doc_id, () => {}); + this.clientA.emit("leaveDoc", this.doc_id, () => {}); + this.clientA.emit("joinDoc", this.doc_id, () => {}); + this.clientA.emit("leaveDoc", this.doc_id, () => {}); return this.clientA.emit("joinDoc", this.doc_id, cb); }, diff --git a/services/real-time/test/acceptance/coffee/ReceiveUpdateTests.js b/services/real-time/test/acceptance/coffee/ReceiveUpdateTests.js index 960ddb145e..d141b27745 100644 --- a/services/real-time/test/acceptance/coffee/ReceiveUpdateTests.js +++ b/services/real-time/test/acceptance/coffee/ReceiveUpdateTests.js @@ -1,3 +1,10 @@ +/* eslint-disable + camelcase, + handle-callback-err, + no-unused-vars, +*/ +// TODO: This file was created by bulk-decaffeinate. +// Fix any style issues and re-enable lint. /* * decaffeinate suggestions: * DS102: Remove unnecessary code created because of implicit returns diff --git a/services/real-time/test/acceptance/coffee/RouterTests.js b/services/real-time/test/acceptance/coffee/RouterTests.js index 6254eb5208..844a4061cf 100644 --- a/services/real-time/test/acceptance/coffee/RouterTests.js +++ b/services/real-time/test/acceptance/coffee/RouterTests.js @@ -1,3 +1,8 @@ +/* eslint-disable + camelcase, +*/ +// TODO: This file was created by bulk-decaffeinate. +// Fix any style issues and re-enable lint. /* * decaffeinate suggestions: * DS102: Remove unnecessary code created because of implicit returns @@ -10,7 +15,7 @@ const RealTimeClient = require("./helpers/RealTimeClient"); const FixturesManager = require("./helpers/FixturesManager"); -describe("Router", () => describe("joinProject", function() { +describe("Router", function() { return describe("joinProject", function() { describe("when there is no callback provided", function() { after(function() { return process.removeListener('unhandledRejection', this.onUnhandled); @@ -50,7 +55,7 @@ describe("Router", () => describe("joinProject", function() { ], done); }); - return it("should keep on going", () => expect('still running').to.exist); + return it("should keep on going", function() { return expect('still running').to.exist; }); }); return describe("when there are too many arguments", function() { @@ -98,4 +103,4 @@ describe("Router", () => describe("joinProject", function() { return expect(this.error.message).to.equal('unexpected arguments'); }); }); -})); +}); }); diff --git a/services/real-time/test/acceptance/coffee/SessionSocketsTests.js b/services/real-time/test/acceptance/coffee/SessionSocketsTests.js index 912e0912e5..93f00cd516 100644 --- a/services/real-time/test/acceptance/coffee/SessionSocketsTests.js +++ b/services/real-time/test/acceptance/coffee/SessionSocketsTests.js @@ -1,3 +1,8 @@ +/* eslint-disable + no-return-assign, +*/ +// TODO: This file was created by bulk-decaffeinate. +// Fix any style issues and re-enable lint. /* * decaffeinate suggestions: * DS102: Remove unnecessary code created because of implicit returns @@ -18,10 +23,10 @@ describe('SessionSockets', function() { }); describe('without cookies', function() { - before(() => RealTimeClient.cookie = null); + before(function() { return RealTimeClient.cookie = null; }); return it('should return a lookup error', function(done) { - return this.checkSocket(function(error) { + return this.checkSocket((error) => { expect(error).to.exist; expect(error.message).to.equal('invalid session'); return done(); @@ -30,10 +35,10 @@ describe('SessionSockets', function() { }); describe('with a different cookie', function() { - before(() => RealTimeClient.cookie = "some.key=someValue"); + before(function() { return RealTimeClient.cookie = "some.key=someValue"; }); return it('should return a lookup error', function(done) { - return this.checkSocket(function(error) { + return this.checkSocket((error) => { expect(error).to.exist; expect(error.message).to.equal('invalid session'); return done(); @@ -43,7 +48,7 @@ describe('SessionSockets', function() { describe('with an invalid cookie', function() { before(function(done) { - RealTimeClient.setSession({}, function(error) { + RealTimeClient.setSession({}, (error) => { if (error) { return done(error); } RealTimeClient.cookie = `${Settings.cookieName}=${ RealTimeClient.cookie.slice(17, 49) @@ -54,7 +59,7 @@ describe('SessionSockets', function() { }); return it('should return a lookup error', function(done) { - return this.checkSocket(function(error) { + return this.checkSocket((error) => { expect(error).to.exist; expect(error.message).to.equal('invalid session'); return done(); @@ -63,10 +68,10 @@ describe('SessionSockets', function() { }); describe('with a valid cookie and no matching session', function() { - before(() => RealTimeClient.cookie = `${Settings.cookieName}=unknownId`); + before(function() { return RealTimeClient.cookie = `${Settings.cookieName}=unknownId`; }); return it('should return a lookup error', function(done) { - return this.checkSocket(function(error) { + return this.checkSocket((error) => { expect(error).to.exist; expect(error.message).to.equal('invalid session'); return done(); @@ -81,7 +86,7 @@ describe('SessionSockets', function() { }); return it('should not return an error', function(done) { - return this.checkSocket(function(error) { + return this.checkSocket((error) => { expect(error).to.not.exist; return done(); }); diff --git a/services/real-time/test/acceptance/coffee/SessionTests.js b/services/real-time/test/acceptance/coffee/SessionTests.js index b8da531875..d9614784f2 100644 --- a/services/real-time/test/acceptance/coffee/SessionTests.js +++ b/services/real-time/test/acceptance/coffee/SessionTests.js @@ -1,3 +1,9 @@ +/* eslint-disable + handle-callback-err, + no-return-assign, +*/ +// TODO: This file was created by bulk-decaffeinate. +// Fix any style issues and re-enable lint. /* * decaffeinate suggestions: * DS101: Remove unnecessary use of Array.from @@ -12,7 +18,7 @@ const { const RealTimeClient = require("./helpers/RealTimeClient"); -describe("Session", () => describe("with an established session", function() { +describe("Session", function() { return describe("with an established session", function() { before(function(done) { this.user_id = "mock-user-id"; RealTimeClient.setSession({ @@ -38,7 +44,7 @@ describe("Session", () => describe("with an established session", function() { return it("should appear in the list of connected clients", function(done) { return RealTimeClient.getConnectedClients((error, clients) => { let included = false; - for (let client of Array.from(clients)) { + for (const client of Array.from(clients)) { if (client.client_id === this.client.socket.sessionid) { included = true; break; @@ -48,4 +54,4 @@ describe("Session", () => describe("with an established session", function() { return done(); }); }); -})); +}); }); diff --git a/services/real-time/test/acceptance/coffee/helpers/FixturesManager.js b/services/real-time/test/acceptance/coffee/helpers/FixturesManager.js index bdae3e01e1..81d9dc40af 100644 --- a/services/real-time/test/acceptance/coffee/helpers/FixturesManager.js +++ b/services/real-time/test/acceptance/coffee/helpers/FixturesManager.js @@ -1,3 +1,9 @@ +/* eslint-disable + camelcase, + handle-callback-err, +*/ +// TODO: This file was created by bulk-decaffeinate. +// Fix any style issues and re-enable lint. /* * decaffeinate suggestions: * DS102: Remove unnecessary code created because of implicit returns diff --git a/services/real-time/test/acceptance/coffee/helpers/MockDocUpdaterServer.js b/services/real-time/test/acceptance/coffee/helpers/MockDocUpdaterServer.js index 675c07a0ed..2ce05c4279 100644 --- a/services/real-time/test/acceptance/coffee/helpers/MockDocUpdaterServer.js +++ b/services/real-time/test/acceptance/coffee/helpers/MockDocUpdaterServer.js @@ -1,3 +1,10 @@ +/* eslint-disable + camelcase, + handle-callback-err, + no-return-assign, +*/ +// TODO: This file was created by bulk-decaffeinate. +// Fix any style issues and re-enable lint. /* * decaffeinate suggestions: * DS102: Remove unnecessary code created because of implicit returns @@ -28,7 +35,7 @@ module.exports = (MockDocUpdaterServer = { const {project_id, doc_id} = req.params; let {fromVersion} = req.query; fromVersion = parseInt(fromVersion, 10); - return MockDocUpdaterServer.getDocument(project_id, doc_id, fromVersion, function(error, data) { + return MockDocUpdaterServer.getDocument(project_id, doc_id, fromVersion, (error, data) => { if (error != null) { return next(error); } return res.json(data); }); @@ -36,7 +43,7 @@ module.exports = (MockDocUpdaterServer = { deleteProjectRequest(req, res, next) { const {project_id} = req.params; - return MockDocUpdaterServer.deleteProject(project_id, function(error) { + return MockDocUpdaterServer.deleteProject(project_id, (error) => { if (error != null) { return next(error); } return res.sendStatus(204); }); @@ -51,10 +58,10 @@ module.exports = (MockDocUpdaterServer = { const app = express(); app.get("/project/:project_id/doc/:doc_id", MockDocUpdaterServer.getDocumentRequest); app.delete("/project/:project_id", MockDocUpdaterServer.deleteProjectRequest); - return app.listen(3003, function(error) { + return app.listen(3003, (error) => { MockDocUpdaterServer.running = true; return callback(error); - }).on("error", function(error) { + }).on("error", (error) => { console.error("error starting MockDocUpdaterServer:", error.message); return process.exit(1); }); diff --git a/services/real-time/test/acceptance/coffee/helpers/MockWebServer.js b/services/real-time/test/acceptance/coffee/helpers/MockWebServer.js index 3fb8db33e7..ea928a42ab 100644 --- a/services/real-time/test/acceptance/coffee/helpers/MockWebServer.js +++ b/services/real-time/test/acceptance/coffee/helpers/MockWebServer.js @@ -1,3 +1,10 @@ +/* eslint-disable + camelcase, + handle-callback-err, + no-return-assign, +*/ +// TODO: This file was created by bulk-decaffeinate. +// Fix any style issues and re-enable lint. /* * decaffeinate suggestions: * DS102: Remove unnecessary code created because of implicit returns @@ -32,7 +39,7 @@ module.exports = (MockWebServer = { if (project_id === 'rate-limited') { return res.status(429).send(); } else { - return MockWebServer.joinProject(project_id, user_id, function(error, project, privilegeLevel) { + return MockWebServer.joinProject(project_id, user_id, (error, project, privilegeLevel) => { if (error != null) { return next(error); } return res.json({ project, @@ -50,10 +57,10 @@ module.exports = (MockWebServer = { } const app = express(); app.post("/project/:project_id/join", MockWebServer.joinProjectRequest); - return app.listen(3000, function(error) { + return app.listen(3000, (error) => { MockWebServer.running = true; return callback(error); - }).on("error", function(error) { + }).on("error", (error) => { console.error("error starting MockWebServer:", error.message); return process.exit(1); }); diff --git a/services/real-time/test/acceptance/coffee/helpers/RealTimeClient.js b/services/real-time/test/acceptance/coffee/helpers/RealTimeClient.js index 5fc8466a2a..c5ad0d3c5b 100644 --- a/services/real-time/test/acceptance/coffee/helpers/RealTimeClient.js +++ b/services/real-time/test/acceptance/coffee/helpers/RealTimeClient.js @@ -1,3 +1,10 @@ +/* eslint-disable + camelcase, + handle-callback-err, + no-return-assign, +*/ +// TODO: This file was created by bulk-decaffeinate. +// Fix any style issues and re-enable lint. /* * decaffeinate suggestions: * DS102: Remove unnecessary code created because of implicit returns @@ -27,7 +34,7 @@ io.util.request = function() { if (Client.cookie != null) { return xhr.setRequestHeader("Cookie", Client.cookie); } - }.bind(this); + }; return xhr; }; @@ -38,7 +45,7 @@ module.exports = (Client = { if (callback == null) { callback = function(error) {}; } const sessionId = uid(24); session.cookie = {}; - return rclient.set("sess:" + sessionId, JSON.stringify(session), function(error) { + return rclient.set("sess:" + sessionId, JSON.stringify(session), (error) => { if (error != null) { return callback(error); } const secret = Settings.security.sessionSecret; const cookieKey = 's:' + signature.sign(sessionId, secret); diff --git a/services/real-time/test/acceptance/coffee/helpers/RealtimeServer.js b/services/real-time/test/acceptance/coffee/helpers/RealtimeServer.js index 7c97b003ef..480836d1dd 100644 --- a/services/real-time/test/acceptance/coffee/helpers/RealtimeServer.js +++ b/services/real-time/test/acceptance/coffee/helpers/RealtimeServer.js @@ -1,3 +1,8 @@ +/* eslint-disable + handle-callback-err, +*/ +// TODO: This file was created by bulk-decaffeinate. +// Fix any style issues and re-enable lint. /* * decaffeinate suggestions: * DS101: Remove unnecessary use of Array.from From db98fdac0a18677e789d8fd17fd98a4cb6e8b4f5 Mon Sep 17 00:00:00 2001 From: decaffeinate Date: Tue, 23 Jun 2020 18:30:37 +0100 Subject: [PATCH 381/491] decaffeinate: rename test/acceptance/coffee to test/acceptance/js --- .../real-time/test/acceptance/{coffee => js}/ApplyUpdateTests.js | 0 .../test/acceptance/{coffee => js}/ClientTrackingTests.js | 0 .../real-time/test/acceptance/{coffee => js}/DrainManagerTests.js | 0 .../real-time/test/acceptance/{coffee => js}/EarlyDisconnect.js | 0 .../test/acceptance/{coffee => js}/HttpControllerTests.js | 0 services/real-time/test/acceptance/{coffee => js}/JoinDocTests.js | 0 .../real-time/test/acceptance/{coffee => js}/JoinProjectTests.js | 0 .../real-time/test/acceptance/{coffee => js}/LeaveDocTests.js | 0 .../real-time/test/acceptance/{coffee => js}/LeaveProjectTests.js | 0 services/real-time/test/acceptance/{coffee => js}/PubSubRace.js | 0 .../test/acceptance/{coffee => js}/ReceiveUpdateTests.js | 0 services/real-time/test/acceptance/{coffee => js}/RouterTests.js | 0 .../test/acceptance/{coffee => js}/SessionSocketsTests.js | 0 services/real-time/test/acceptance/{coffee => js}/SessionTests.js | 0 .../test/acceptance/{coffee => js}/helpers/FixturesManager.js | 0 .../acceptance/{coffee => js}/helpers/MockDocUpdaterServer.js | 0 .../test/acceptance/{coffee => js}/helpers/MockWebServer.js | 0 .../test/acceptance/{coffee => js}/helpers/RealTimeClient.js | 0 .../test/acceptance/{coffee => js}/helpers/RealtimeServer.js | 0 19 files changed, 0 insertions(+), 0 deletions(-) rename services/real-time/test/acceptance/{coffee => js}/ApplyUpdateTests.js (100%) rename services/real-time/test/acceptance/{coffee => js}/ClientTrackingTests.js (100%) rename services/real-time/test/acceptance/{coffee => js}/DrainManagerTests.js (100%) rename services/real-time/test/acceptance/{coffee => js}/EarlyDisconnect.js (100%) rename services/real-time/test/acceptance/{coffee => js}/HttpControllerTests.js (100%) rename services/real-time/test/acceptance/{coffee => js}/JoinDocTests.js (100%) rename services/real-time/test/acceptance/{coffee => js}/JoinProjectTests.js (100%) rename services/real-time/test/acceptance/{coffee => js}/LeaveDocTests.js (100%) rename services/real-time/test/acceptance/{coffee => js}/LeaveProjectTests.js (100%) rename services/real-time/test/acceptance/{coffee => js}/PubSubRace.js (100%) rename services/real-time/test/acceptance/{coffee => js}/ReceiveUpdateTests.js (100%) rename services/real-time/test/acceptance/{coffee => js}/RouterTests.js (100%) rename services/real-time/test/acceptance/{coffee => js}/SessionSocketsTests.js (100%) rename services/real-time/test/acceptance/{coffee => js}/SessionTests.js (100%) rename services/real-time/test/acceptance/{coffee => js}/helpers/FixturesManager.js (100%) rename services/real-time/test/acceptance/{coffee => js}/helpers/MockDocUpdaterServer.js (100%) rename services/real-time/test/acceptance/{coffee => js}/helpers/MockWebServer.js (100%) rename services/real-time/test/acceptance/{coffee => js}/helpers/RealTimeClient.js (100%) rename services/real-time/test/acceptance/{coffee => js}/helpers/RealtimeServer.js (100%) diff --git a/services/real-time/test/acceptance/coffee/ApplyUpdateTests.js b/services/real-time/test/acceptance/js/ApplyUpdateTests.js similarity index 100% rename from services/real-time/test/acceptance/coffee/ApplyUpdateTests.js rename to services/real-time/test/acceptance/js/ApplyUpdateTests.js diff --git a/services/real-time/test/acceptance/coffee/ClientTrackingTests.js b/services/real-time/test/acceptance/js/ClientTrackingTests.js similarity index 100% rename from services/real-time/test/acceptance/coffee/ClientTrackingTests.js rename to services/real-time/test/acceptance/js/ClientTrackingTests.js diff --git a/services/real-time/test/acceptance/coffee/DrainManagerTests.js b/services/real-time/test/acceptance/js/DrainManagerTests.js similarity index 100% rename from services/real-time/test/acceptance/coffee/DrainManagerTests.js rename to services/real-time/test/acceptance/js/DrainManagerTests.js diff --git a/services/real-time/test/acceptance/coffee/EarlyDisconnect.js b/services/real-time/test/acceptance/js/EarlyDisconnect.js similarity index 100% rename from services/real-time/test/acceptance/coffee/EarlyDisconnect.js rename to services/real-time/test/acceptance/js/EarlyDisconnect.js diff --git a/services/real-time/test/acceptance/coffee/HttpControllerTests.js b/services/real-time/test/acceptance/js/HttpControllerTests.js similarity index 100% rename from services/real-time/test/acceptance/coffee/HttpControllerTests.js rename to services/real-time/test/acceptance/js/HttpControllerTests.js diff --git a/services/real-time/test/acceptance/coffee/JoinDocTests.js b/services/real-time/test/acceptance/js/JoinDocTests.js similarity index 100% rename from services/real-time/test/acceptance/coffee/JoinDocTests.js rename to services/real-time/test/acceptance/js/JoinDocTests.js diff --git a/services/real-time/test/acceptance/coffee/JoinProjectTests.js b/services/real-time/test/acceptance/js/JoinProjectTests.js similarity index 100% rename from services/real-time/test/acceptance/coffee/JoinProjectTests.js rename to services/real-time/test/acceptance/js/JoinProjectTests.js diff --git a/services/real-time/test/acceptance/coffee/LeaveDocTests.js b/services/real-time/test/acceptance/js/LeaveDocTests.js similarity index 100% rename from services/real-time/test/acceptance/coffee/LeaveDocTests.js rename to services/real-time/test/acceptance/js/LeaveDocTests.js diff --git a/services/real-time/test/acceptance/coffee/LeaveProjectTests.js b/services/real-time/test/acceptance/js/LeaveProjectTests.js similarity index 100% rename from services/real-time/test/acceptance/coffee/LeaveProjectTests.js rename to services/real-time/test/acceptance/js/LeaveProjectTests.js diff --git a/services/real-time/test/acceptance/coffee/PubSubRace.js b/services/real-time/test/acceptance/js/PubSubRace.js similarity index 100% rename from services/real-time/test/acceptance/coffee/PubSubRace.js rename to services/real-time/test/acceptance/js/PubSubRace.js diff --git a/services/real-time/test/acceptance/coffee/ReceiveUpdateTests.js b/services/real-time/test/acceptance/js/ReceiveUpdateTests.js similarity index 100% rename from services/real-time/test/acceptance/coffee/ReceiveUpdateTests.js rename to services/real-time/test/acceptance/js/ReceiveUpdateTests.js diff --git a/services/real-time/test/acceptance/coffee/RouterTests.js b/services/real-time/test/acceptance/js/RouterTests.js similarity index 100% rename from services/real-time/test/acceptance/coffee/RouterTests.js rename to services/real-time/test/acceptance/js/RouterTests.js diff --git a/services/real-time/test/acceptance/coffee/SessionSocketsTests.js b/services/real-time/test/acceptance/js/SessionSocketsTests.js similarity index 100% rename from services/real-time/test/acceptance/coffee/SessionSocketsTests.js rename to services/real-time/test/acceptance/js/SessionSocketsTests.js diff --git a/services/real-time/test/acceptance/coffee/SessionTests.js b/services/real-time/test/acceptance/js/SessionTests.js similarity index 100% rename from services/real-time/test/acceptance/coffee/SessionTests.js rename to services/real-time/test/acceptance/js/SessionTests.js diff --git a/services/real-time/test/acceptance/coffee/helpers/FixturesManager.js b/services/real-time/test/acceptance/js/helpers/FixturesManager.js similarity index 100% rename from services/real-time/test/acceptance/coffee/helpers/FixturesManager.js rename to services/real-time/test/acceptance/js/helpers/FixturesManager.js diff --git a/services/real-time/test/acceptance/coffee/helpers/MockDocUpdaterServer.js b/services/real-time/test/acceptance/js/helpers/MockDocUpdaterServer.js similarity index 100% rename from services/real-time/test/acceptance/coffee/helpers/MockDocUpdaterServer.js rename to services/real-time/test/acceptance/js/helpers/MockDocUpdaterServer.js diff --git a/services/real-time/test/acceptance/coffee/helpers/MockWebServer.js b/services/real-time/test/acceptance/js/helpers/MockWebServer.js similarity index 100% rename from services/real-time/test/acceptance/coffee/helpers/MockWebServer.js rename to services/real-time/test/acceptance/js/helpers/MockWebServer.js diff --git a/services/real-time/test/acceptance/coffee/helpers/RealTimeClient.js b/services/real-time/test/acceptance/js/helpers/RealTimeClient.js similarity index 100% rename from services/real-time/test/acceptance/coffee/helpers/RealTimeClient.js rename to services/real-time/test/acceptance/js/helpers/RealTimeClient.js diff --git a/services/real-time/test/acceptance/coffee/helpers/RealtimeServer.js b/services/real-time/test/acceptance/js/helpers/RealtimeServer.js similarity index 100% rename from services/real-time/test/acceptance/coffee/helpers/RealtimeServer.js rename to services/real-time/test/acceptance/js/helpers/RealtimeServer.js From 8a7fc78dc845c97add480311b748b4bd05a4f932 Mon Sep 17 00:00:00 2001 From: decaffeinate Date: Tue, 23 Jun 2020 18:30:45 +0100 Subject: [PATCH 382/491] prettier: convert test/acceptance decaffeinated files to Prettier format --- .../test/acceptance/js/ApplyUpdateTests.js | 701 +++++++++------ .../test/acceptance/js/ClientTrackingTests.js | 413 +++++---- .../test/acceptance/js/DrainManagerTests.js | 188 ++-- .../test/acceptance/js/EarlyDisconnect.js | 427 +++++---- .../test/acceptance/js/HttpControllerTests.js | 175 ++-- .../test/acceptance/js/JoinDocTests.js | 841 +++++++++++------- .../test/acceptance/js/JoinProjectTests.js | 334 ++++--- .../test/acceptance/js/LeaveDocTests.js | 257 +++--- .../test/acceptance/js/LeaveProjectTests.js | 403 +++++---- .../test/acceptance/js/PubSubRace.js | 570 +++++++----- .../test/acceptance/js/ReceiveUpdateTests.js | 544 ++++++----- .../test/acceptance/js/RouterTests.js | 185 ++-- .../test/acceptance/js/SessionSocketsTests.js | 156 ++-- .../test/acceptance/js/SessionTests.js | 86 +- .../acceptance/js/helpers/FixturesManager.js | 166 ++-- .../js/helpers/MockDocUpdaterServer.js | 132 +-- .../acceptance/js/helpers/MockWebServer.js | 123 +-- .../acceptance/js/helpers/RealTimeClient.js | 190 ++-- .../acceptance/js/helpers/RealtimeServer.js | 77 +- 19 files changed, 3463 insertions(+), 2505 deletions(-) diff --git a/services/real-time/test/acceptance/js/ApplyUpdateTests.js b/services/real-time/test/acceptance/js/ApplyUpdateTests.js index bc9d315b0f..2c5b753f29 100644 --- a/services/real-time/test/acceptance/js/ApplyUpdateTests.js +++ b/services/real-time/test/acceptance/js/ApplyUpdateTests.js @@ -12,313 +12,436 @@ * DS201: Simplify complex destructure assignments * Full docs: https://github.com/decaffeinate/decaffeinate/blob/master/docs/suggestions.md */ -const async = require("async"); -const chai = require("chai"); -const { - expect -} = chai; -chai.should(); +const async = require('async') +const chai = require('chai') +const { expect } = chai +chai.should() -const RealTimeClient = require("./helpers/RealTimeClient"); -const FixturesManager = require("./helpers/FixturesManager"); +const RealTimeClient = require('./helpers/RealTimeClient') +const FixturesManager = require('./helpers/FixturesManager') -const settings = require("settings-sharelatex"); -const redis = require("redis-sharelatex"); -const rclient = redis.createClient(settings.redis.documentupdater); +const settings = require('settings-sharelatex') +const redis = require('redis-sharelatex') +const rclient = redis.createClient(settings.redis.documentupdater) -const redisSettings = settings.redis; +const redisSettings = settings.redis -describe("applyOtUpdate", function() { - before(function() { - return this.update = { - op: [{i: "foo", p: 42}] - };}); - describe("when authorized", function() { - before(function(done) { - return async.series([ - cb => { - return FixturesManager.setUpProject({ - privilegeLevel: "readAndWrite" - }, (e, {project_id, user_id}) => { - this.project_id = project_id; - this.user_id = user_id; - return cb(e); - }); - }, - - cb => { - return FixturesManager.setUpDoc(this.project_id, {lines: this.lines, version: this.version, ops: this.ops}, (e, {doc_id}) => { - this.doc_id = doc_id; - return cb(e); - }); - }, +describe('applyOtUpdate', function () { + before(function () { + return (this.update = { + op: [{ i: 'foo', p: 42 }] + }) + }) + describe('when authorized', function () { + before(function (done) { + return async.series( + [ + (cb) => { + return FixturesManager.setUpProject( + { + privilegeLevel: 'readAndWrite' + }, + (e, { project_id, user_id }) => { + this.project_id = project_id + this.user_id = user_id + return cb(e) + } + ) + }, - cb => { - this.client = RealTimeClient.connect(); - return this.client.on("connectionAccepted", cb); - }, - - cb => { - return this.client.emit("joinProject", {project_id: this.project_id}, cb); - }, - - cb => { - return this.client.emit("joinDoc", this.doc_id, cb); - }, - - cb => { - return this.client.emit("applyOtUpdate", this.doc_id, this.update, cb); - } - ], done); - }); - - it("should push the doc into the pending updates list", function(done) { - rclient.lrange("pending-updates-list", 0, -1, (error, ...rest) => { - const [doc_id] = Array.from(rest[0]); - doc_id.should.equal(`${this.project_id}:${this.doc_id}`); - return done(); - }); - return null; - }); + (cb) => { + return FixturesManager.setUpDoc( + this.project_id, + { lines: this.lines, version: this.version, ops: this.ops }, + (e, { doc_id }) => { + this.doc_id = doc_id + return cb(e) + } + ) + }, - it("should push the update into redis", function(done) { - rclient.lrange(redisSettings.documentupdater.key_schema.pendingUpdates({doc_id: this.doc_id}), 0, -1, (error, ...rest) => { - let [update] = Array.from(rest[0]); - update = JSON.parse(update); - update.op.should.deep.equal(this.update.op); - update.meta.should.deep.equal({ - source: this.client.publicId, - user_id: this.user_id - }); - return done(); - }); - return null; - }); + (cb) => { + this.client = RealTimeClient.connect() + return this.client.on('connectionAccepted', cb) + }, - return after(function(done) { - return async.series([ - cb => rclient.del("pending-updates-list", cb), - cb => rclient.del("DocsWithPendingUpdates", `${this.project_id}:${this.doc_id}`, cb), - cb => rclient.del(redisSettings.documentupdater.key_schema.pendingUpdates(this.doc_id), cb) - ], done); - }); - }); - - describe("when authorized with a huge edit update", function() { - before(function(done) { - this.update = { - op: { - p: 12, - t: "update is too large".repeat(1024 * 400) // >7MB - } - }; - return async.series([ - cb => { - return FixturesManager.setUpProject({ - privilegeLevel: "readAndWrite" - }, (e, {project_id, user_id}) => { - this.project_id = project_id; - this.user_id = user_id; - return cb(e); - }); - }, + (cb) => { + return this.client.emit( + 'joinProject', + { project_id: this.project_id }, + cb + ) + }, - cb => { - return FixturesManager.setUpDoc(this.project_id, {lines: this.lines, version: this.version, ops: this.ops}, (e, {doc_id}) => { - this.doc_id = doc_id; - return cb(e); - }); - }, + (cb) => { + return this.client.emit('joinDoc', this.doc_id, cb) + }, - cb => { - this.client = RealTimeClient.connect(); - this.client.on("connectionAccepted", cb); - return this.client.on("otUpdateError", otUpdateError => { - this.otUpdateError = otUpdateError; - - }); - }, + (cb) => { + return this.client.emit( + 'applyOtUpdate', + this.doc_id, + this.update, + cb + ) + } + ], + done + ) + }) - cb => { - return this.client.emit("joinProject", {project_id: this.project_id}, cb); - }, + it('should push the doc into the pending updates list', function (done) { + rclient.lrange('pending-updates-list', 0, -1, (error, ...rest) => { + const [doc_id] = Array.from(rest[0]) + doc_id.should.equal(`${this.project_id}:${this.doc_id}`) + return done() + }) + return null + }) - cb => { - return this.client.emit("joinDoc", this.doc_id, cb); - }, + it('should push the update into redis', function (done) { + rclient.lrange( + redisSettings.documentupdater.key_schema.pendingUpdates({ + doc_id: this.doc_id + }), + 0, + -1, + (error, ...rest) => { + let [update] = Array.from(rest[0]) + update = JSON.parse(update) + update.op.should.deep.equal(this.update.op) + update.meta.should.deep.equal({ + source: this.client.publicId, + user_id: this.user_id + }) + return done() + } + ) + return null + }) - cb => { - return this.client.emit("applyOtUpdate", this.doc_id, this.update, error => { - this.error = error; - return cb(); - }); - } - ], done); - }); + return after(function (done) { + return async.series( + [ + (cb) => rclient.del('pending-updates-list', cb), + (cb) => + rclient.del( + 'DocsWithPendingUpdates', + `${this.project_id}:${this.doc_id}`, + cb + ), + (cb) => + rclient.del( + redisSettings.documentupdater.key_schema.pendingUpdates( + this.doc_id + ), + cb + ) + ], + done + ) + }) + }) - it("should not return an error", function() { - return expect(this.error).to.not.exist; - }); + describe('when authorized with a huge edit update', function () { + before(function (done) { + this.update = { + op: { + p: 12, + t: 'update is too large'.repeat(1024 * 400) // >7MB + } + } + return async.series( + [ + (cb) => { + return FixturesManager.setUpProject( + { + privilegeLevel: 'readAndWrite' + }, + (e, { project_id, user_id }) => { + this.project_id = project_id + this.user_id = user_id + return cb(e) + } + ) + }, - it("should send an otUpdateError to the client", function(done) { - return setTimeout(() => { - expect(this.otUpdateError).to.exist; - return done(); - } - , 300); - }); + (cb) => { + return FixturesManager.setUpDoc( + this.project_id, + { lines: this.lines, version: this.version, ops: this.ops }, + (e, { doc_id }) => { + this.doc_id = doc_id + return cb(e) + } + ) + }, - it("should disconnect the client", function(done) { - return setTimeout(() => { - this.client.socket.connected.should.equal(false); - return done(); - } - , 300); - }); + (cb) => { + this.client = RealTimeClient.connect() + this.client.on('connectionAccepted', cb) + return this.client.on('otUpdateError', (otUpdateError) => { + this.otUpdateError = otUpdateError + }) + }, - return it("should not put the update in redis", function(done) { - rclient.llen(redisSettings.documentupdater.key_schema.pendingUpdates({doc_id: this.doc_id}), (error, len) => { - len.should.equal(0); - return done(); - }); - return null; - }); - }); + (cb) => { + return this.client.emit( + 'joinProject', + { project_id: this.project_id }, + cb + ) + }, - describe("when authorized to read-only with an edit update", function() { - before(function(done) { - return async.series([ - cb => { - return FixturesManager.setUpProject({ - privilegeLevel: "readOnly" - }, (e, {project_id, user_id}) => { - this.project_id = project_id; - this.user_id = user_id; - return cb(e); - }); - }, - - cb => { - return FixturesManager.setUpDoc(this.project_id, {lines: this.lines, version: this.version, ops: this.ops}, (e, {doc_id}) => { - this.doc_id = doc_id; - return cb(e); - }); - }, + (cb) => { + return this.client.emit('joinDoc', this.doc_id, cb) + }, - cb => { - this.client = RealTimeClient.connect(); - return this.client.on("connectionAccepted", cb); - }, - - cb => { - return this.client.emit("joinProject", {project_id: this.project_id}, cb); - }, - - cb => { - return this.client.emit("joinDoc", this.doc_id, cb); - }, - - cb => { - return this.client.emit("applyOtUpdate", this.doc_id, this.update, error => { - this.error = error; - return cb(); - }); - } - ], done); - }); - - it("should return an error", function() { - return expect(this.error).to.exist; - }); - - it("should disconnect the client", function(done) { - return setTimeout(() => { - this.client.socket.connected.should.equal(false); - return done(); - } - , 300); - }); - - return it("should not put the update in redis", function(done) { - rclient.llen(redisSettings.documentupdater.key_schema.pendingUpdates({doc_id: this.doc_id}), (error, len) => { - len.should.equal(0); - return done(); - }); - return null; - }); - }); - - return describe("when authorized to read-only with a comment update", function() { - before(function(done) { - this.comment_update = { - op: [{c: "foo", p: 42}] - }; - return async.series([ - cb => { - return FixturesManager.setUpProject({ - privilegeLevel: "readOnly" - }, (e, {project_id, user_id}) => { - this.project_id = project_id; - this.user_id = user_id; - return cb(e); - }); - }, - - cb => { - return FixturesManager.setUpDoc(this.project_id, {lines: this.lines, version: this.version, ops: this.ops}, (e, {doc_id}) => { - this.doc_id = doc_id; - return cb(e); - }); - }, + (cb) => { + return this.client.emit( + 'applyOtUpdate', + this.doc_id, + this.update, + (error) => { + this.error = error + return cb() + } + ) + } + ], + done + ) + }) - cb => { - this.client = RealTimeClient.connect(); - return this.client.on("connectionAccepted", cb); - }, - - cb => { - return this.client.emit("joinProject", {project_id: this.project_id}, cb); - }, - - cb => { - return this.client.emit("joinDoc", this.doc_id, cb); - }, - - cb => { - return this.client.emit("applyOtUpdate", this.doc_id, this.comment_update, cb); - } - ], done); - }); - - it("should push the doc into the pending updates list", function(done) { - rclient.lrange("pending-updates-list", 0, -1, (error, ...rest) => { - const [doc_id] = Array.from(rest[0]); - doc_id.should.equal(`${this.project_id}:${this.doc_id}`); - return done(); - }); - return null; - }); + it('should not return an error', function () { + return expect(this.error).to.not.exist + }) - it("should push the update into redis", function(done) { - rclient.lrange(redisSettings.documentupdater.key_schema.pendingUpdates({doc_id: this.doc_id}), 0, -1, (error, ...rest) => { - let [update] = Array.from(rest[0]); - update = JSON.parse(update); - update.op.should.deep.equal(this.comment_update.op); - update.meta.should.deep.equal({ - source: this.client.publicId, - user_id: this.user_id - }); - return done(); - }); - return null; - }); + it('should send an otUpdateError to the client', function (done) { + return setTimeout(() => { + expect(this.otUpdateError).to.exist + return done() + }, 300) + }) - return after(function(done) { - return async.series([ - cb => rclient.del("pending-updates-list", cb), - cb => rclient.del("DocsWithPendingUpdates", `${this.project_id}:${this.doc_id}`, cb), - cb => rclient.del(redisSettings.documentupdater.key_schema.pendingUpdates({doc_id: this.doc_id}), cb) - ], done); - }); - }); -}); + it('should disconnect the client', function (done) { + return setTimeout(() => { + this.client.socket.connected.should.equal(false) + return done() + }, 300) + }) + + return it('should not put the update in redis', function (done) { + rclient.llen( + redisSettings.documentupdater.key_schema.pendingUpdates({ + doc_id: this.doc_id + }), + (error, len) => { + len.should.equal(0) + return done() + } + ) + return null + }) + }) + + describe('when authorized to read-only with an edit update', function () { + before(function (done) { + return async.series( + [ + (cb) => { + return FixturesManager.setUpProject( + { + privilegeLevel: 'readOnly' + }, + (e, { project_id, user_id }) => { + this.project_id = project_id + this.user_id = user_id + return cb(e) + } + ) + }, + + (cb) => { + return FixturesManager.setUpDoc( + this.project_id, + { lines: this.lines, version: this.version, ops: this.ops }, + (e, { doc_id }) => { + this.doc_id = doc_id + return cb(e) + } + ) + }, + + (cb) => { + this.client = RealTimeClient.connect() + return this.client.on('connectionAccepted', cb) + }, + + (cb) => { + return this.client.emit( + 'joinProject', + { project_id: this.project_id }, + cb + ) + }, + + (cb) => { + return this.client.emit('joinDoc', this.doc_id, cb) + }, + + (cb) => { + return this.client.emit( + 'applyOtUpdate', + this.doc_id, + this.update, + (error) => { + this.error = error + return cb() + } + ) + } + ], + done + ) + }) + + it('should return an error', function () { + return expect(this.error).to.exist + }) + + it('should disconnect the client', function (done) { + return setTimeout(() => { + this.client.socket.connected.should.equal(false) + return done() + }, 300) + }) + + return it('should not put the update in redis', function (done) { + rclient.llen( + redisSettings.documentupdater.key_schema.pendingUpdates({ + doc_id: this.doc_id + }), + (error, len) => { + len.should.equal(0) + return done() + } + ) + return null + }) + }) + + return describe('when authorized to read-only with a comment update', function () { + before(function (done) { + this.comment_update = { + op: [{ c: 'foo', p: 42 }] + } + return async.series( + [ + (cb) => { + return FixturesManager.setUpProject( + { + privilegeLevel: 'readOnly' + }, + (e, { project_id, user_id }) => { + this.project_id = project_id + this.user_id = user_id + return cb(e) + } + ) + }, + + (cb) => { + return FixturesManager.setUpDoc( + this.project_id, + { lines: this.lines, version: this.version, ops: this.ops }, + (e, { doc_id }) => { + this.doc_id = doc_id + return cb(e) + } + ) + }, + + (cb) => { + this.client = RealTimeClient.connect() + return this.client.on('connectionAccepted', cb) + }, + + (cb) => { + return this.client.emit( + 'joinProject', + { project_id: this.project_id }, + cb + ) + }, + + (cb) => { + return this.client.emit('joinDoc', this.doc_id, cb) + }, + + (cb) => { + return this.client.emit( + 'applyOtUpdate', + this.doc_id, + this.comment_update, + cb + ) + } + ], + done + ) + }) + + it('should push the doc into the pending updates list', function (done) { + rclient.lrange('pending-updates-list', 0, -1, (error, ...rest) => { + const [doc_id] = Array.from(rest[0]) + doc_id.should.equal(`${this.project_id}:${this.doc_id}`) + return done() + }) + return null + }) + + it('should push the update into redis', function (done) { + rclient.lrange( + redisSettings.documentupdater.key_schema.pendingUpdates({ + doc_id: this.doc_id + }), + 0, + -1, + (error, ...rest) => { + let [update] = Array.from(rest[0]) + update = JSON.parse(update) + update.op.should.deep.equal(this.comment_update.op) + update.meta.should.deep.equal({ + source: this.client.publicId, + user_id: this.user_id + }) + return done() + } + ) + return null + }) + + return after(function (done) { + return async.series( + [ + (cb) => rclient.del('pending-updates-list', cb), + (cb) => + rclient.del( + 'DocsWithPendingUpdates', + `${this.project_id}:${this.doc_id}`, + cb + ), + (cb) => + rclient.del( + redisSettings.documentupdater.key_schema.pendingUpdates({ + doc_id: this.doc_id + }), + cb + ) + ], + done + ) + }) + }) +}) diff --git a/services/real-time/test/acceptance/js/ClientTrackingTests.js b/services/real-time/test/acceptance/js/ClientTrackingTests.js index 75e23d0719..079baadb58 100644 --- a/services/real-time/test/acceptance/js/ClientTrackingTests.js +++ b/services/real-time/test/acceptance/js/ClientTrackingTests.js @@ -12,187 +12,244 @@ * DS207: Consider shorter variations of null checks * Full docs: https://github.com/decaffeinate/decaffeinate/blob/master/docs/suggestions.md */ -const chai = require("chai"); -const { - expect -} = chai; -chai.should(); +const chai = require('chai') +const { expect } = chai +chai.should() -const RealTimeClient = require("./helpers/RealTimeClient"); -const MockWebServer = require("./helpers/MockWebServer"); -const FixturesManager = require("./helpers/FixturesManager"); +const RealTimeClient = require('./helpers/RealTimeClient') +const MockWebServer = require('./helpers/MockWebServer') +const FixturesManager = require('./helpers/FixturesManager') -const async = require("async"); +const async = require('async') -describe("clientTracking", function() { - describe("when a client updates its cursor location", function() { - before(function(done) { - return async.series([ - cb => { - return FixturesManager.setUpProject({ - privilegeLevel: "owner", - project: { name: "Test Project" } - }, (error, {user_id, project_id}) => { this.user_id = user_id; this.project_id = project_id; return cb(); }); - }, - - cb => { - return FixturesManager.setUpDoc(this.project_id, {lines: this.lines, version: this.version, ops: this.ops}, (e, {doc_id}) => { - this.doc_id = doc_id; - return cb(e); - }); - }, - - cb => { - this.clientA = RealTimeClient.connect(); - return this.clientA.on("connectionAccepted", cb); - }, - - cb => { - this.clientB = RealTimeClient.connect(); - return this.clientB.on("connectionAccepted", cb); - }, - - cb => { - return this.clientA.emit("joinProject", { - project_id: this.project_id - }, cb); - }, - - cb => { - return this.clientA.emit("joinDoc", this.doc_id, cb); - }, - - cb => { - return this.clientB.emit("joinProject", { - project_id: this.project_id - }, cb); - }, - - cb => { - this.updates = []; - this.clientB.on("clientTracking.clientUpdated", data => { - return this.updates.push(data); - }); +describe('clientTracking', function () { + describe('when a client updates its cursor location', function () { + before(function (done) { + return async.series( + [ + (cb) => { + return FixturesManager.setUpProject( + { + privilegeLevel: 'owner', + project: { name: 'Test Project' } + }, + (error, { user_id, project_id }) => { + this.user_id = user_id + this.project_id = project_id + return cb() + } + ) + }, - return this.clientA.emit("clientTracking.updatePosition", { - row: (this.row = 42), - column: (this.column = 36), - doc_id: this.doc_id - }, (error) => { - if (error != null) { throw error; } - return setTimeout(cb, 300); - }); - } // Give the message a chance to reach client B. - ], done); - }); - - it("should tell other clients about the update", function() { - return this.updates.should.deep.equal([ - { - row: this.row, - column: this.column, - doc_id: this.doc_id, - id: this.clientA.publicId, - user_id: this.user_id, - name: "Joe Bloggs" - } - ]); - }); - - return it("should record the update in getConnectedUsers", function(done) { - return this.clientB.emit("clientTracking.getConnectedUsers", (error, users) => { - for (const user of Array.from(users)) { - if (user.client_id === this.clientA.publicId) { - expect(user.cursorData).to.deep.equal({ - row: this.row, - column: this.column, - doc_id: this.doc_id - }); - return done(); - } - } - throw new Error("user was never found"); - }); - }); - }); - - return describe("when an anonymous client updates its cursor location", function() { - before(function(done) { - return async.series([ - cb => { - return FixturesManager.setUpProject({ - privilegeLevel: "owner", - project: { name: "Test Project" }, - publicAccess: "readAndWrite" - }, (error, {user_id, project_id}) => { this.user_id = user_id; this.project_id = project_id; return cb(); }); - }, - - cb => { - return FixturesManager.setUpDoc(this.project_id, {lines: this.lines, version: this.version, ops: this.ops}, (e, {doc_id}) => { - this.doc_id = doc_id; - return cb(e); - }); - }, - - cb => { - this.clientA = RealTimeClient.connect(); - return this.clientA.on("connectionAccepted", cb); - }, + (cb) => { + return FixturesManager.setUpDoc( + this.project_id, + { lines: this.lines, version: this.version, ops: this.ops }, + (e, { doc_id }) => { + this.doc_id = doc_id + return cb(e) + } + ) + }, - cb => { - return this.clientA.emit("joinProject", { - project_id: this.project_id - }, cb); - }, - - cb => { - return RealTimeClient.setSession({}, cb); - }, - - cb => { - this.anonymous = RealTimeClient.connect(); - return this.anonymous.on("connectionAccepted", cb); - }, - - cb => { - return this.anonymous.emit("joinProject", { - project_id: this.project_id - }, cb); - }, - - cb => { - return this.anonymous.emit("joinDoc", this.doc_id, cb); - }, - - cb => { - this.updates = []; - this.clientA.on("clientTracking.clientUpdated", data => { - return this.updates.push(data); - }); + (cb) => { + this.clientA = RealTimeClient.connect() + return this.clientA.on('connectionAccepted', cb) + }, - return this.anonymous.emit("clientTracking.updatePosition", { - row: (this.row = 42), - column: (this.column = 36), - doc_id: this.doc_id - }, (error) => { - if (error != null) { throw error; } - return setTimeout(cb, 300); - }); - } // Give the message a chance to reach client B. - ], done); - }); - - return it("should tell other clients about the update", function() { - return this.updates.should.deep.equal([ - { - row: this.row, - column: this.column, - doc_id: this.doc_id, - id: this.anonymous.publicId, - user_id: "anonymous-user", - name: "" - } - ]); - }); -}); -}); + (cb) => { + this.clientB = RealTimeClient.connect() + return this.clientB.on('connectionAccepted', cb) + }, + + (cb) => { + return this.clientA.emit( + 'joinProject', + { + project_id: this.project_id + }, + cb + ) + }, + + (cb) => { + return this.clientA.emit('joinDoc', this.doc_id, cb) + }, + + (cb) => { + return this.clientB.emit( + 'joinProject', + { + project_id: this.project_id + }, + cb + ) + }, + + (cb) => { + this.updates = [] + this.clientB.on('clientTracking.clientUpdated', (data) => { + return this.updates.push(data) + }) + + return this.clientA.emit( + 'clientTracking.updatePosition', + { + row: (this.row = 42), + column: (this.column = 36), + doc_id: this.doc_id + }, + (error) => { + if (error != null) { + throw error + } + return setTimeout(cb, 300) + } + ) + } // Give the message a chance to reach client B. + ], + done + ) + }) + + it('should tell other clients about the update', function () { + return this.updates.should.deep.equal([ + { + row: this.row, + column: this.column, + doc_id: this.doc_id, + id: this.clientA.publicId, + user_id: this.user_id, + name: 'Joe Bloggs' + } + ]) + }) + + return it('should record the update in getConnectedUsers', function (done) { + return this.clientB.emit( + 'clientTracking.getConnectedUsers', + (error, users) => { + for (const user of Array.from(users)) { + if (user.client_id === this.clientA.publicId) { + expect(user.cursorData).to.deep.equal({ + row: this.row, + column: this.column, + doc_id: this.doc_id + }) + return done() + } + } + throw new Error('user was never found') + } + ) + }) + }) + + return describe('when an anonymous client updates its cursor location', function () { + before(function (done) { + return async.series( + [ + (cb) => { + return FixturesManager.setUpProject( + { + privilegeLevel: 'owner', + project: { name: 'Test Project' }, + publicAccess: 'readAndWrite' + }, + (error, { user_id, project_id }) => { + this.user_id = user_id + this.project_id = project_id + return cb() + } + ) + }, + + (cb) => { + return FixturesManager.setUpDoc( + this.project_id, + { lines: this.lines, version: this.version, ops: this.ops }, + (e, { doc_id }) => { + this.doc_id = doc_id + return cb(e) + } + ) + }, + + (cb) => { + this.clientA = RealTimeClient.connect() + return this.clientA.on('connectionAccepted', cb) + }, + + (cb) => { + return this.clientA.emit( + 'joinProject', + { + project_id: this.project_id + }, + cb + ) + }, + + (cb) => { + return RealTimeClient.setSession({}, cb) + }, + + (cb) => { + this.anonymous = RealTimeClient.connect() + return this.anonymous.on('connectionAccepted', cb) + }, + + (cb) => { + return this.anonymous.emit( + 'joinProject', + { + project_id: this.project_id + }, + cb + ) + }, + + (cb) => { + return this.anonymous.emit('joinDoc', this.doc_id, cb) + }, + + (cb) => { + this.updates = [] + this.clientA.on('clientTracking.clientUpdated', (data) => { + return this.updates.push(data) + }) + + return this.anonymous.emit( + 'clientTracking.updatePosition', + { + row: (this.row = 42), + column: (this.column = 36), + doc_id: this.doc_id + }, + (error) => { + if (error != null) { + throw error + } + return setTimeout(cb, 300) + } + ) + } // Give the message a chance to reach client B. + ], + done + ) + }) + + return it('should tell other clients about the update', function () { + return this.updates.should.deep.equal([ + { + row: this.row, + column: this.column, + doc_id: this.doc_id, + id: this.anonymous.publicId, + user_id: 'anonymous-user', + name: '' + } + ]) + }) + }) +}) diff --git a/services/real-time/test/acceptance/js/DrainManagerTests.js b/services/real-time/test/acceptance/js/DrainManagerTests.js index 1ff4a4afbb..d312d34aa9 100644 --- a/services/real-time/test/acceptance/js/DrainManagerTests.js +++ b/services/real-time/test/acceptance/js/DrainManagerTests.js @@ -8,98 +8,128 @@ * DS102: Remove unnecessary code created because of implicit returns * Full docs: https://github.com/decaffeinate/decaffeinate/blob/master/docs/suggestions.md */ -const RealTimeClient = require("./helpers/RealTimeClient"); -const FixturesManager = require("./helpers/FixturesManager"); +const RealTimeClient = require('./helpers/RealTimeClient') +const FixturesManager = require('./helpers/FixturesManager') -const { - expect -} = require("chai"); +const { expect } = require('chai') -const async = require("async"); -const request = require("request"); +const async = require('async') +const request = require('request') -const Settings = require("settings-sharelatex"); +const Settings = require('settings-sharelatex') -const drain = function(rate, callback) { - request.post({ - url: `http://localhost:3026/drain?rate=${rate}`, - auth: { - user: Settings.internal.realTime.user, - pass: Settings.internal.realTime.pass - } - }, (error, response, data) => callback(error, data)); - return null; -}; +const drain = function (rate, callback) { + request.post( + { + url: `http://localhost:3026/drain?rate=${rate}`, + auth: { + user: Settings.internal.realTime.user, + pass: Settings.internal.realTime.pass + } + }, + (error, response, data) => callback(error, data) + ) + return null +} -describe("DrainManagerTests", function() { - before(function(done) { - FixturesManager.setUpProject({ - privilegeLevel: "owner", - project: { - name: "Test Project" - } - }, (e, {project_id, user_id}) => { this.project_id = project_id; this.user_id = user_id; return done(); }); - return null; - }); +describe('DrainManagerTests', function () { + before(function (done) { + FixturesManager.setUpProject( + { + privilegeLevel: 'owner', + project: { + name: 'Test Project' + } + }, + (e, { project_id, user_id }) => { + this.project_id = project_id + this.user_id = user_id + return done() + } + ) + return null + }) - before(function(done) { - // cleanup to speedup reconnecting - this.timeout(10000); - return RealTimeClient.disconnectAllClients(done); - }); + before(function (done) { + // cleanup to speedup reconnecting + this.timeout(10000) + return RealTimeClient.disconnectAllClients(done) + }) - // trigger and check cleanup - it("should have disconnected all previous clients", function(done) { return RealTimeClient.getConnectedClients((error, data) => { - if (error) { return done(error); } - expect(data.length).to.equal(0); - return done(); - }); }); + // trigger and check cleanup + it('should have disconnected all previous clients', function (done) { + return RealTimeClient.getConnectedClients((error, data) => { + if (error) { + return done(error) + } + expect(data.length).to.equal(0) + return done() + }) + }) - return describe("with two clients in the project", function() { - beforeEach(function(done) { - return async.series([ - cb => { - this.clientA = RealTimeClient.connect(); - return this.clientA.on("connectionAccepted", cb); - }, + return describe('with two clients in the project', function () { + beforeEach(function (done) { + return async.series( + [ + (cb) => { + this.clientA = RealTimeClient.connect() + return this.clientA.on('connectionAccepted', cb) + }, - cb => { - this.clientB = RealTimeClient.connect(); - return this.clientB.on("connectionAccepted", cb); - }, + (cb) => { + this.clientB = RealTimeClient.connect() + return this.clientB.on('connectionAccepted', cb) + }, - cb => { - return this.clientA.emit("joinProject", {project_id: this.project_id}, cb); - }, + (cb) => { + return this.clientA.emit( + 'joinProject', + { project_id: this.project_id }, + cb + ) + }, - cb => { - return this.clientB.emit("joinProject", {project_id: this.project_id}, cb); - } - ], done); - }); + (cb) => { + return this.clientB.emit( + 'joinProject', + { project_id: this.project_id }, + cb + ) + } + ], + done + ) + }) - return describe("starting to drain", function() { - beforeEach(function(done) { - return async.parallel([ - cb => { - return this.clientA.on("reconnectGracefully", cb); - }, - cb => { - return this.clientB.on("reconnectGracefully", cb); - }, + return describe('starting to drain', function () { + beforeEach(function (done) { + return async.parallel( + [ + (cb) => { + return this.clientA.on('reconnectGracefully', cb) + }, + (cb) => { + return this.clientB.on('reconnectGracefully', cb) + }, - cb => drain(2, cb) - ], done); - }); + (cb) => drain(2, cb) + ], + done + ) + }) - afterEach(function(done) { return drain(0, done); }); // reset drain + afterEach(function (done) { + return drain(0, done) + }) // reset drain - it("should not timeout", function() { return expect(true).to.equal(true); }); + it('should not timeout', function () { + return expect(true).to.equal(true) + }) - return it("should not have disconnected", function() { - expect(this.clientA.socket.connected).to.equal(true); - return expect(this.clientB.socket.connected).to.equal(true); - }); - }); - }); -}); + return it('should not have disconnected', function () { + expect(this.clientA.socket.connected).to.equal(true) + return expect(this.clientB.socket.connected).to.equal(true) + }) + }) + }) +}) diff --git a/services/real-time/test/acceptance/js/EarlyDisconnect.js b/services/real-time/test/acceptance/js/EarlyDisconnect.js index 427d77040b..25e8fbc427 100644 --- a/services/real-time/test/acceptance/js/EarlyDisconnect.js +++ b/services/real-time/test/acceptance/js/EarlyDisconnect.js @@ -10,206 +10,279 @@ * DS102: Remove unnecessary code created because of implicit returns * Full docs: https://github.com/decaffeinate/decaffeinate/blob/master/docs/suggestions.md */ -const async = require("async"); -const {expect} = require("chai"); +const async = require('async') +const { expect } = require('chai') -const RealTimeClient = require("./helpers/RealTimeClient"); -const MockDocUpdaterServer = require("./helpers/MockDocUpdaterServer"); -const MockWebServer = require("./helpers/MockWebServer"); -const FixturesManager = require("./helpers/FixturesManager"); +const RealTimeClient = require('./helpers/RealTimeClient') +const MockDocUpdaterServer = require('./helpers/MockDocUpdaterServer') +const MockWebServer = require('./helpers/MockWebServer') +const FixturesManager = require('./helpers/FixturesManager') -const settings = require("settings-sharelatex"); -const redis = require("redis-sharelatex"); -const rclient = redis.createClient(settings.redis.pubsub); -const rclientRT = redis.createClient(settings.redis.realtime); -const KeysRT = settings.redis.realtime.key_schema; +const settings = require('settings-sharelatex') +const redis = require('redis-sharelatex') +const rclient = redis.createClient(settings.redis.pubsub) +const rclientRT = redis.createClient(settings.redis.realtime) +const KeysRT = settings.redis.realtime.key_schema -describe("EarlyDisconnect", function() { - before(function(done) { return MockDocUpdaterServer.run(done); }); +describe('EarlyDisconnect', function () { + before(function (done) { + return MockDocUpdaterServer.run(done) + }) - describe("when the client disconnects before joinProject completes", function() { - before(function() { - // slow down web-api requests to force the race condition - let joinProject; - this.actualWebAPIjoinProject = (joinProject = MockWebServer.joinProject); - return MockWebServer.joinProject = (project_id, user_id, cb) => setTimeout(() => joinProject(project_id, user_id, cb) - , 300); - }); + describe('when the client disconnects before joinProject completes', function () { + before(function () { + // slow down web-api requests to force the race condition + let joinProject + this.actualWebAPIjoinProject = joinProject = MockWebServer.joinProject + return (MockWebServer.joinProject = (project_id, user_id, cb) => + setTimeout(() => joinProject(project_id, user_id, cb), 300)) + }) - after(function() { - return MockWebServer.joinProject = this.actualWebAPIjoinProject; - }); + after(function () { + return (MockWebServer.joinProject = this.actualWebAPIjoinProject) + }) - beforeEach(function(done) { - return async.series([ - cb => { - return FixturesManager.setUpProject({ - privilegeLevel: "owner", - project: { - name: "Test Project" - } - }, (e, {project_id, user_id}) => { this.project_id = project_id; this.user_id = user_id; return cb(); }); - }, + beforeEach(function (done) { + return async.series( + [ + (cb) => { + return FixturesManager.setUpProject( + { + privilegeLevel: 'owner', + project: { + name: 'Test Project' + } + }, + (e, { project_id, user_id }) => { + this.project_id = project_id + this.user_id = user_id + return cb() + } + ) + }, - cb => { - this.clientA = RealTimeClient.connect(); - return this.clientA.on("connectionAccepted", cb); - }, + (cb) => { + this.clientA = RealTimeClient.connect() + return this.clientA.on('connectionAccepted', cb) + }, - cb => { - this.clientA.emit("joinProject", {project_id: this.project_id}, (() => {})); - // disconnect before joinProject completes - this.clientA.on("disconnect", () => cb()); - return this.clientA.disconnect(); - }, + (cb) => { + this.clientA.emit( + 'joinProject', + { project_id: this.project_id }, + () => {} + ) + // disconnect before joinProject completes + this.clientA.on('disconnect', () => cb()) + return this.clientA.disconnect() + }, - cb => { - // wait for joinDoc and subscribe - return setTimeout(cb, 500); - } - ], done); - }); + (cb) => { + // wait for joinDoc and subscribe + return setTimeout(cb, 500) + } + ], + done + ) + }) - // we can force the race condition, there is no need to repeat too often - return Array.from(Array.from({length: 5}).map((_, i) => i+1)).map((attempt) => - it(`should not subscribe to the pub/sub channel anymore (race ${attempt})`, function(done) { - rclient.pubsub('CHANNELS', (err, resp) => { - if (err) { return done(err); } - expect(resp).to.not.include(`editor-events:${this.project_id}`); - return done(); - }); - return null; - })); - }); + // we can force the race condition, there is no need to repeat too often + return Array.from(Array.from({ length: 5 }).map((_, i) => i + 1)).map( + (attempt) => + it(`should not subscribe to the pub/sub channel anymore (race ${attempt})`, function (done) { + rclient.pubsub('CHANNELS', (err, resp) => { + if (err) { + return done(err) + } + expect(resp).to.not.include(`editor-events:${this.project_id}`) + return done() + }) + return null + }) + ) + }) - describe("when the client disconnects before joinDoc completes", function() { - beforeEach(function(done) { - return async.series([ - cb => { - return FixturesManager.setUpProject({ - privilegeLevel: "owner", - project: { - name: "Test Project" - } - }, (e, {project_id, user_id}) => { this.project_id = project_id; this.user_id = user_id; return cb(); }); - }, + describe('when the client disconnects before joinDoc completes', function () { + beforeEach(function (done) { + return async.series( + [ + (cb) => { + return FixturesManager.setUpProject( + { + privilegeLevel: 'owner', + project: { + name: 'Test Project' + } + }, + (e, { project_id, user_id }) => { + this.project_id = project_id + this.user_id = user_id + return cb() + } + ) + }, - cb => { - this.clientA = RealTimeClient.connect(); - return this.clientA.on("connectionAccepted", cb); - }, + (cb) => { + this.clientA = RealTimeClient.connect() + return this.clientA.on('connectionAccepted', cb) + }, - cb => { - return this.clientA.emit("joinProject", {project_id: this.project_id}, (error, project, privilegeLevel, protocolVersion) => { - this.project = project; - this.privilegeLevel = privilegeLevel; - this.protocolVersion = protocolVersion; - return cb(error); - }); - }, + (cb) => { + return this.clientA.emit( + 'joinProject', + { project_id: this.project_id }, + (error, project, privilegeLevel, protocolVersion) => { + this.project = project + this.privilegeLevel = privilegeLevel + this.protocolVersion = protocolVersion + return cb(error) + } + ) + }, - cb => { - return FixturesManager.setUpDoc(this.project_id, {lines: this.lines, version: this.version, ops: this.ops}, (e, {doc_id}) => { - this.doc_id = doc_id; - return cb(e); - }); - }, + (cb) => { + return FixturesManager.setUpDoc( + this.project_id, + { lines: this.lines, version: this.version, ops: this.ops }, + (e, { doc_id }) => { + this.doc_id = doc_id + return cb(e) + } + ) + }, - cb => { - this.clientA.emit("joinDoc", this.doc_id, (() => {})); - // disconnect before joinDoc completes - this.clientA.on("disconnect", () => cb()); - return this.clientA.disconnect(); - }, + (cb) => { + this.clientA.emit('joinDoc', this.doc_id, () => {}) + // disconnect before joinDoc completes + this.clientA.on('disconnect', () => cb()) + return this.clientA.disconnect() + }, - cb => { - // wait for subscribe and unsubscribe - return setTimeout(cb, 100); - } - ], done); - }); + (cb) => { + // wait for subscribe and unsubscribe + return setTimeout(cb, 100) + } + ], + done + ) + }) - // we can not force the race condition, so we have to try many times - return Array.from(Array.from({length: 20}).map((_, i) => i+1)).map((attempt) => - it(`should not subscribe to the pub/sub channels anymore (race ${attempt})`, function(done) { - rclient.pubsub('CHANNELS', (err, resp) => { - if (err) { return done(err); } - expect(resp).to.not.include(`editor-events:${this.project_id}`); + // we can not force the race condition, so we have to try many times + return Array.from(Array.from({ length: 20 }).map((_, i) => i + 1)).map( + (attempt) => + it(`should not subscribe to the pub/sub channels anymore (race ${attempt})`, function (done) { + rclient.pubsub('CHANNELS', (err, resp) => { + if (err) { + return done(err) + } + expect(resp).to.not.include(`editor-events:${this.project_id}`) - return rclient.pubsub('CHANNELS', (err, resp) => { - if (err) { return done(err); } - expect(resp).to.not.include(`applied-ops:${this.doc_id}`); - return done(); - }); - }); - return null; - })); - }); + return rclient.pubsub('CHANNELS', (err, resp) => { + if (err) { + return done(err) + } + expect(resp).to.not.include(`applied-ops:${this.doc_id}`) + return done() + }) + }) + return null + }) + ) + }) - return describe("when the client disconnects before clientTracking.updatePosition starts", function() { - beforeEach(function(done) { - return async.series([ - cb => { - return FixturesManager.setUpProject({ - privilegeLevel: "owner", - project: { - name: "Test Project" - } - }, (e, {project_id, user_id}) => { this.project_id = project_id; this.user_id = user_id; return cb(); }); - }, + return describe('when the client disconnects before clientTracking.updatePosition starts', function () { + beforeEach(function (done) { + return async.series( + [ + (cb) => { + return FixturesManager.setUpProject( + { + privilegeLevel: 'owner', + project: { + name: 'Test Project' + } + }, + (e, { project_id, user_id }) => { + this.project_id = project_id + this.user_id = user_id + return cb() + } + ) + }, - cb => { - this.clientA = RealTimeClient.connect(); - return this.clientA.on("connectionAccepted", cb); - }, + (cb) => { + this.clientA = RealTimeClient.connect() + return this.clientA.on('connectionAccepted', cb) + }, - cb => { - return this.clientA.emit("joinProject", {project_id: this.project_id}, (error, project, privilegeLevel, protocolVersion) => { - this.project = project; - this.privilegeLevel = privilegeLevel; - this.protocolVersion = protocolVersion; - return cb(error); - }); - }, + (cb) => { + return this.clientA.emit( + 'joinProject', + { project_id: this.project_id }, + (error, project, privilegeLevel, protocolVersion) => { + this.project = project + this.privilegeLevel = privilegeLevel + this.protocolVersion = protocolVersion + return cb(error) + } + ) + }, - cb => { - return FixturesManager.setUpDoc(this.project_id, {lines: this.lines, version: this.version, ops: this.ops}, (e, {doc_id}) => { - this.doc_id = doc_id; - return cb(e); - }); - }, + (cb) => { + return FixturesManager.setUpDoc( + this.project_id, + { lines: this.lines, version: this.version, ops: this.ops }, + (e, { doc_id }) => { + this.doc_id = doc_id + return cb(e) + } + ) + }, - cb => { - return this.clientA.emit("joinDoc", this.doc_id, cb); - }, + (cb) => { + return this.clientA.emit('joinDoc', this.doc_id, cb) + }, - cb => { - this.clientA.emit("clientTracking.updatePosition", { - row: 42, - column: 36, - doc_id: this.doc_id - }, (() => {})); - // disconnect before updateClientPosition completes - this.clientA.on("disconnect", () => cb()); - return this.clientA.disconnect(); - }, + (cb) => { + this.clientA.emit( + 'clientTracking.updatePosition', + { + row: 42, + column: 36, + doc_id: this.doc_id + }, + () => {} + ) + // disconnect before updateClientPosition completes + this.clientA.on('disconnect', () => cb()) + return this.clientA.disconnect() + }, - cb => { - // wait for updateClientPosition - return setTimeout(cb, 100); - } - ], done); - }); + (cb) => { + // wait for updateClientPosition + return setTimeout(cb, 100) + } + ], + done + ) + }) - // we can not force the race condition, so we have to try many times - return Array.from(Array.from({length: 20}).map((_, i) => i+1)).map((attempt) => - it(`should not show the client as connected (race ${attempt})`, function(done) { - rclientRT.smembers(KeysRT.clientsInProject({project_id: this.project_id}), (err, results) => { - if (err) { return done(err); } - expect(results).to.deep.equal([]); - return done(); - }); - return null; - })); - }); -}); + // we can not force the race condition, so we have to try many times + return Array.from(Array.from({ length: 20 }).map((_, i) => i + 1)).map( + (attempt) => + it(`should not show the client as connected (race ${attempt})`, function (done) { + rclientRT.smembers( + KeysRT.clientsInProject({ project_id: this.project_id }), + (err, results) => { + if (err) { + return done(err) + } + expect(results).to.deep.equal([]) + return done() + } + ) + return null + }) + ) + }) +}) diff --git a/services/real-time/test/acceptance/js/HttpControllerTests.js b/services/real-time/test/acceptance/js/HttpControllerTests.js index c701a91e20..5d40a30def 100644 --- a/services/real-time/test/acceptance/js/HttpControllerTests.js +++ b/services/real-time/test/acceptance/js/HttpControllerTests.js @@ -8,89 +8,110 @@ * DS102: Remove unnecessary code created because of implicit returns * Full docs: https://github.com/decaffeinate/decaffeinate/blob/master/docs/suggestions.md */ -const async = require('async'); -const { - expect -} = require('chai'); +const async = require('async') +const { expect } = require('chai') const request = require('request').defaults({ - baseUrl: 'http://localhost:3026' -}); + baseUrl: 'http://localhost:3026' +}) -const RealTimeClient = require("./helpers/RealTimeClient"); -const FixturesManager = require("./helpers/FixturesManager"); +const RealTimeClient = require('./helpers/RealTimeClient') +const FixturesManager = require('./helpers/FixturesManager') -describe('HttpControllerTests', function() { - describe('without a user', function() { return it('should return 404 for the client view', function(done) { - const client_id = 'not-existing'; - return request.get({ - url: `/clients/${client_id}`, - json: true - }, (error, response, data) => { - if (error) { return done(error); } - expect(response.statusCode).to.equal(404); - return done(); - }); - }); }); +describe('HttpControllerTests', function () { + describe('without a user', function () { + return it('should return 404 for the client view', function (done) { + const client_id = 'not-existing' + return request.get( + { + url: `/clients/${client_id}`, + json: true + }, + (error, response, data) => { + if (error) { + return done(error) + } + expect(response.statusCode).to.equal(404) + return done() + } + ) + }) + }) - return describe('with a user and after joining a project', function() { - before(function(done) { - return async.series([ - cb => { - return FixturesManager.setUpProject({ - privilegeLevel: "owner" - }, (error, {project_id, user_id}) => { - this.project_id = project_id; - this.user_id = user_id; - return cb(error); - }); - }, + return describe('with a user and after joining a project', function () { + before(function (done) { + return async.series( + [ + (cb) => { + return FixturesManager.setUpProject( + { + privilegeLevel: 'owner' + }, + (error, { project_id, user_id }) => { + this.project_id = project_id + this.user_id = user_id + return cb(error) + } + ) + }, - cb => { - return FixturesManager.setUpDoc(this.project_id, {}, (error, {doc_id}) => { - this.doc_id = doc_id; - return cb(error); - }); - }, + (cb) => { + return FixturesManager.setUpDoc( + this.project_id, + {}, + (error, { doc_id }) => { + this.doc_id = doc_id + return cb(error) + } + ) + }, - cb => { - this.client = RealTimeClient.connect(); - return this.client.on("connectionAccepted", cb); - }, + (cb) => { + this.client = RealTimeClient.connect() + return this.client.on('connectionAccepted', cb) + }, - cb => { - return this.client.emit("joinProject", {project_id: this.project_id}, cb); - }, + (cb) => { + return this.client.emit( + 'joinProject', + { project_id: this.project_id }, + cb + ) + }, - cb => { - return this.client.emit("joinDoc", this.doc_id, cb); - } - ], done); - }); + (cb) => { + return this.client.emit('joinDoc', this.doc_id, cb) + } + ], + done + ) + }) - return it('should send a client view', function(done) { - return request.get({ - url: `/clients/${this.client.socket.sessionid}`, - json: true - }, (error, response, data) => { - if (error) { return done(error); } - expect(response.statusCode).to.equal(200); - expect(data.connected_time).to.exist; - delete data.connected_time; - // .email is not set in the session - delete data.email; - expect(data).to.deep.equal({ - client_id: this.client.socket.sessionid, - first_name: 'Joe', - last_name: 'Bloggs', - project_id: this.project_id, - user_id: this.user_id, - rooms: [ - this.project_id, - this.doc_id, - ] - }); - return done(); - }); - }); - }); -}); + return it('should send a client view', function (done) { + return request.get( + { + url: `/clients/${this.client.socket.sessionid}`, + json: true + }, + (error, response, data) => { + if (error) { + return done(error) + } + expect(response.statusCode).to.equal(200) + expect(data.connected_time).to.exist + delete data.connected_time + // .email is not set in the session + delete data.email + expect(data).to.deep.equal({ + client_id: this.client.socket.sessionid, + first_name: 'Joe', + last_name: 'Bloggs', + project_id: this.project_id, + user_id: this.user_id, + rooms: [this.project_id, this.doc_id] + }) + return done() + } + ) + }) + }) +}) diff --git a/services/real-time/test/acceptance/js/JoinDocTests.js b/services/real-time/test/acceptance/js/JoinDocTests.js index f55af76820..217731a1df 100644 --- a/services/real-time/test/acceptance/js/JoinDocTests.js +++ b/services/real-time/test/acceptance/js/JoinDocTests.js @@ -11,348 +11,555 @@ * DS102: Remove unnecessary code created because of implicit returns * Full docs: https://github.com/decaffeinate/decaffeinate/blob/master/docs/suggestions.md */ -const chai = require("chai"); -const { - expect -} = chai; -chai.should(); +const chai = require('chai') +const { expect } = chai +chai.should() -const RealTimeClient = require("./helpers/RealTimeClient"); -const MockDocUpdaterServer = require("./helpers/MockDocUpdaterServer"); -const FixturesManager = require("./helpers/FixturesManager"); +const RealTimeClient = require('./helpers/RealTimeClient') +const MockDocUpdaterServer = require('./helpers/MockDocUpdaterServer') +const FixturesManager = require('./helpers/FixturesManager') -const async = require("async"); +const async = require('async') -describe("joinDoc", function() { - before(function() { - this.lines = ["test", "doc", "lines"]; - this.version = 42; - this.ops = ["mock", "doc", "ops"]; - return this.ranges = {"mock": "ranges"};}); - - describe("when authorised readAndWrite", function() { - before(function(done) { - return async.series([ - cb => { - return FixturesManager.setUpProject({ - privilegeLevel: "readAndWrite" - }, (e, {project_id, user_id}) => { - this.project_id = project_id; - this.user_id = user_id; - return cb(e); - }); - }, - - cb => { - return FixturesManager.setUpDoc(this.project_id, {lines: this.lines, version: this.version, ops: this.ops, ranges: this.ranges}, (e, {doc_id}) => { - this.doc_id = doc_id; - return cb(e); - }); - }, +describe('joinDoc', function () { + before(function () { + this.lines = ['test', 'doc', 'lines'] + this.version = 42 + this.ops = ['mock', 'doc', 'ops'] + return (this.ranges = { mock: 'ranges' }) + }) - cb => { - this.client = RealTimeClient.connect(); - return this.client.on("connectionAccepted", cb); - }, - - cb => { - return this.client.emit("joinProject", {project_id: this.project_id}, cb); - }, - - cb => { - return this.client.emit("joinDoc", this.doc_id, (error, ...rest) => { [...this.returnedArgs] = Array.from(rest); return cb(error); }); - } - ], done); - }); + describe('when authorised readAndWrite', function () { + before(function (done) { + return async.series( + [ + (cb) => { + return FixturesManager.setUpProject( + { + privilegeLevel: 'readAndWrite' + }, + (e, { project_id, user_id }) => { + this.project_id = project_id + this.user_id = user_id + return cb(e) + } + ) + }, - it("should get the doc from the doc updater", function() { - return MockDocUpdaterServer.getDocument - .calledWith(this.project_id, this.doc_id, -1) - .should.equal(true); - }); - - it("should return the doc lines, version, ranges and ops", function() { - return this.returnedArgs.should.deep.equal([this.lines, this.version, this.ops, this.ranges]); - }); - - return it("should have joined the doc room", function(done) { - return RealTimeClient.getConnectedClient(this.client.socket.sessionid, (error, client) => { - expect(Array.from(client.rooms).includes(this.doc_id)).to.equal(true); - return done(); - }); - }); - }); - - describe("when authorised readOnly", function() { - before(function(done) { - return async.series([ - cb => { - return FixturesManager.setUpProject({ - privilegeLevel: "readOnly" - }, (e, {project_id, user_id}) => { - this.project_id = project_id; - this.user_id = user_id; - return cb(e); - }); - }, - - cb => { - return FixturesManager.setUpDoc(this.project_id, {lines: this.lines, version: this.version, ops: this.ops, ranges: this.ranges}, (e, {doc_id}) => { - this.doc_id = doc_id; - return cb(e); - }); - }, + (cb) => { + return FixturesManager.setUpDoc( + this.project_id, + { + lines: this.lines, + version: this.version, + ops: this.ops, + ranges: this.ranges + }, + (e, { doc_id }) => { + this.doc_id = doc_id + return cb(e) + } + ) + }, - cb => { - this.client = RealTimeClient.connect(); - return this.client.on("connectionAccepted", cb); - }, - - cb => { - return this.client.emit("joinProject", {project_id: this.project_id}, cb); - }, - - cb => { - return this.client.emit("joinDoc", this.doc_id, (error, ...rest) => { [...this.returnedArgs] = Array.from(rest); return cb(error); }); - } - ], done); - }); + (cb) => { + this.client = RealTimeClient.connect() + return this.client.on('connectionAccepted', cb) + }, - it("should get the doc from the doc updater", function() { - return MockDocUpdaterServer.getDocument - .calledWith(this.project_id, this.doc_id, -1) - .should.equal(true); - }); - - it("should return the doc lines, version, ranges and ops", function() { - return this.returnedArgs.should.deep.equal([this.lines, this.version, this.ops, this.ranges]); - }); - - return it("should have joined the doc room", function(done) { - return RealTimeClient.getConnectedClient(this.client.socket.sessionid, (error, client) => { - expect(Array.from(client.rooms).includes(this.doc_id)).to.equal(true); - return done(); - }); - }); - }); - - describe("when authorised as owner", function() { - before(function(done) { - return async.series([ - cb => { - return FixturesManager.setUpProject({ - privilegeLevel: "owner" - }, (e, {project_id, user_id}) => { - this.project_id = project_id; - this.user_id = user_id; - return cb(e); - }); - }, - - cb => { - return FixturesManager.setUpDoc(this.project_id, {lines: this.lines, version: this.version, ops: this.ops, ranges: this.ranges}, (e, {doc_id}) => { - this.doc_id = doc_id; - return cb(e); - }); - }, + (cb) => { + return this.client.emit( + 'joinProject', + { project_id: this.project_id }, + cb + ) + }, - cb => { - this.client = RealTimeClient.connect(); - return this.client.on("connectionAccepted", cb); - }, - - cb => { - return this.client.emit("joinProject", {project_id: this.project_id}, cb); - }, - - cb => { - return this.client.emit("joinDoc", this.doc_id, (error, ...rest) => { [...this.returnedArgs] = Array.from(rest); return cb(error); }); - } - ], done); - }); + (cb) => { + return this.client.emit( + 'joinDoc', + this.doc_id, + (error, ...rest) => { + ;[...this.returnedArgs] = Array.from(rest) + return cb(error) + } + ) + } + ], + done + ) + }) - it("should get the doc from the doc updater", function() { - return MockDocUpdaterServer.getDocument - .calledWith(this.project_id, this.doc_id, -1) - .should.equal(true); - }); - - it("should return the doc lines, version, ranges and ops", function() { - return this.returnedArgs.should.deep.equal([this.lines, this.version, this.ops, this.ranges]); - }); - - return it("should have joined the doc room", function(done) { - return RealTimeClient.getConnectedClient(this.client.socket.sessionid, (error, client) => { - expect(Array.from(client.rooms).includes(this.doc_id)).to.equal(true); - return done(); - }); - }); - }); + it('should get the doc from the doc updater', function () { + return MockDocUpdaterServer.getDocument + .calledWith(this.project_id, this.doc_id, -1) + .should.equal(true) + }) - // It is impossible to write an acceptance test to test joining an unauthorized - // project, since joinProject already catches that. If you can join a project, - // then you can join a doc in that project. - - describe("with a fromVersion", function() { - before(function(done) { - this.fromVersion = 36; - return async.series([ - cb => { - return FixturesManager.setUpProject({ - privilegeLevel: "readAndWrite" - }, (e, {project_id, user_id}) => { - this.project_id = project_id; - this.user_id = user_id; - return cb(e); - }); - }, - - cb => { - return FixturesManager.setUpDoc(this.project_id, {lines: this.lines, version: this.version, ops: this.ops, ranges: this.ranges}, (e, {doc_id}) => { - this.doc_id = doc_id; - return cb(e); - }); - }, + it('should return the doc lines, version, ranges and ops', function () { + return this.returnedArgs.should.deep.equal([ + this.lines, + this.version, + this.ops, + this.ranges + ]) + }) - cb => { - this.client = RealTimeClient.connect(); - return this.client.on("connectionAccepted", cb); - }, - - cb => { - return this.client.emit("joinProject", {project_id: this.project_id}, cb); - }, - - cb => { - return this.client.emit("joinDoc", this.doc_id, this.fromVersion, (error, ...rest) => { [...this.returnedArgs] = Array.from(rest); return cb(error); }); - } - ], done); - }); + return it('should have joined the doc room', function (done) { + return RealTimeClient.getConnectedClient( + this.client.socket.sessionid, + (error, client) => { + expect(Array.from(client.rooms).includes(this.doc_id)).to.equal(true) + return done() + } + ) + }) + }) - it("should get the doc from the doc updater with the fromVersion", function() { - return MockDocUpdaterServer.getDocument - .calledWith(this.project_id, this.doc_id, this.fromVersion) - .should.equal(true); - }); - - it("should return the doc lines, version, ranges and ops", function() { - return this.returnedArgs.should.deep.equal([this.lines, this.version, this.ops, this.ranges]); - }); - - return it("should have joined the doc room", function(done) { - return RealTimeClient.getConnectedClient(this.client.socket.sessionid, (error, client) => { - expect(Array.from(client.rooms).includes(this.doc_id)).to.equal(true); - return done(); - }); - }); - }); + describe('when authorised readOnly', function () { + before(function (done) { + return async.series( + [ + (cb) => { + return FixturesManager.setUpProject( + { + privilegeLevel: 'readOnly' + }, + (e, { project_id, user_id }) => { + this.project_id = project_id + this.user_id = user_id + return cb(e) + } + ) + }, - describe("with options", function() { - before(function(done) { - this.options = { encodeRanges: true }; - return async.series([ - cb => { - return FixturesManager.setUpProject({ - privilegeLevel: "readAndWrite" - }, (e, {project_id, user_id}) => { - this.project_id = project_id; - this.user_id = user_id; - return cb(e); - }); - }, + (cb) => { + return FixturesManager.setUpDoc( + this.project_id, + { + lines: this.lines, + version: this.version, + ops: this.ops, + ranges: this.ranges + }, + (e, { doc_id }) => { + this.doc_id = doc_id + return cb(e) + } + ) + }, - cb => { - return FixturesManager.setUpDoc(this.project_id, {lines: this.lines, version: this.version, ops: this.ops, ranges: this.ranges}, (e, {doc_id}) => { - this.doc_id = doc_id; - return cb(e); - }); - }, + (cb) => { + this.client = RealTimeClient.connect() + return this.client.on('connectionAccepted', cb) + }, - cb => { - this.client = RealTimeClient.connect(); - return this.client.on("connectionAccepted", cb); - }, + (cb) => { + return this.client.emit( + 'joinProject', + { project_id: this.project_id }, + cb + ) + }, - cb => { - return this.client.emit("joinProject", {project_id: this.project_id}, cb); - }, + (cb) => { + return this.client.emit( + 'joinDoc', + this.doc_id, + (error, ...rest) => { + ;[...this.returnedArgs] = Array.from(rest) + return cb(error) + } + ) + } + ], + done + ) + }) - cb => { - return this.client.emit("joinDoc", this.doc_id, this.options, (error, ...rest) => { [...this.returnedArgs] = Array.from(rest); return cb(error); }); - } - ], done); - }); + it('should get the doc from the doc updater', function () { + return MockDocUpdaterServer.getDocument + .calledWith(this.project_id, this.doc_id, -1) + .should.equal(true) + }) - it("should get the doc from the doc updater with the default fromVersion", function() { - return MockDocUpdaterServer.getDocument - .calledWith(this.project_id, this.doc_id, -1) - .should.equal(true); - }); + it('should return the doc lines, version, ranges and ops', function () { + return this.returnedArgs.should.deep.equal([ + this.lines, + this.version, + this.ops, + this.ranges + ]) + }) - it("should return the doc lines, version, ranges and ops", function() { - return this.returnedArgs.should.deep.equal([this.lines, this.version, this.ops, this.ranges]); - }); + return it('should have joined the doc room', function (done) { + return RealTimeClient.getConnectedClient( + this.client.socket.sessionid, + (error, client) => { + expect(Array.from(client.rooms).includes(this.doc_id)).to.equal(true) + return done() + } + ) + }) + }) - return it("should have joined the doc room", function(done) { - return RealTimeClient.getConnectedClient(this.client.socket.sessionid, (error, client) => { - expect(Array.from(client.rooms).includes(this.doc_id)).to.equal(true); - return done(); - }); - }); - }); + describe('when authorised as owner', function () { + before(function (done) { + return async.series( + [ + (cb) => { + return FixturesManager.setUpProject( + { + privilegeLevel: 'owner' + }, + (e, { project_id, user_id }) => { + this.project_id = project_id + this.user_id = user_id + return cb(e) + } + ) + }, - return describe("with fromVersion and options", function() { - before(function(done) { - this.fromVersion = 36; - this.options = { encodeRanges: true }; - return async.series([ - cb => { - return FixturesManager.setUpProject({ - privilegeLevel: "readAndWrite" - }, (e, {project_id, user_id}) => { - this.project_id = project_id; - this.user_id = user_id; - return cb(e); - }); - }, + (cb) => { + return FixturesManager.setUpDoc( + this.project_id, + { + lines: this.lines, + version: this.version, + ops: this.ops, + ranges: this.ranges + }, + (e, { doc_id }) => { + this.doc_id = doc_id + return cb(e) + } + ) + }, - cb => { - return FixturesManager.setUpDoc(this.project_id, {lines: this.lines, version: this.version, ops: this.ops, ranges: this.ranges}, (e, {doc_id}) => { - this.doc_id = doc_id; - return cb(e); - }); - }, + (cb) => { + this.client = RealTimeClient.connect() + return this.client.on('connectionAccepted', cb) + }, - cb => { - this.client = RealTimeClient.connect(); - return this.client.on("connectionAccepted", cb); - }, + (cb) => { + return this.client.emit( + 'joinProject', + { project_id: this.project_id }, + cb + ) + }, - cb => { - return this.client.emit("joinProject", {project_id: this.project_id}, cb); - }, + (cb) => { + return this.client.emit( + 'joinDoc', + this.doc_id, + (error, ...rest) => { + ;[...this.returnedArgs] = Array.from(rest) + return cb(error) + } + ) + } + ], + done + ) + }) - cb => { - return this.client.emit("joinDoc", this.doc_id, this.fromVersion, this.options, (error, ...rest) => { [...this.returnedArgs] = Array.from(rest); return cb(error); }); - } - ], done); - }); + it('should get the doc from the doc updater', function () { + return MockDocUpdaterServer.getDocument + .calledWith(this.project_id, this.doc_id, -1) + .should.equal(true) + }) - it("should get the doc from the doc updater with the fromVersion", function() { - return MockDocUpdaterServer.getDocument - .calledWith(this.project_id, this.doc_id, this.fromVersion) - .should.equal(true); - }); + it('should return the doc lines, version, ranges and ops', function () { + return this.returnedArgs.should.deep.equal([ + this.lines, + this.version, + this.ops, + this.ranges + ]) + }) - it("should return the doc lines, version, ranges and ops", function() { - return this.returnedArgs.should.deep.equal([this.lines, this.version, this.ops, this.ranges]); - }); + return it('should have joined the doc room', function (done) { + return RealTimeClient.getConnectedClient( + this.client.socket.sessionid, + (error, client) => { + expect(Array.from(client.rooms).includes(this.doc_id)).to.equal(true) + return done() + } + ) + }) + }) - return it("should have joined the doc room", function(done) { - return RealTimeClient.getConnectedClient(this.client.socket.sessionid, (error, client) => { - expect(Array.from(client.rooms).includes(this.doc_id)).to.equal(true); - return done(); - }); - }); - }); -}); + // It is impossible to write an acceptance test to test joining an unauthorized + // project, since joinProject already catches that. If you can join a project, + // then you can join a doc in that project. + + describe('with a fromVersion', function () { + before(function (done) { + this.fromVersion = 36 + return async.series( + [ + (cb) => { + return FixturesManager.setUpProject( + { + privilegeLevel: 'readAndWrite' + }, + (e, { project_id, user_id }) => { + this.project_id = project_id + this.user_id = user_id + return cb(e) + } + ) + }, + + (cb) => { + return FixturesManager.setUpDoc( + this.project_id, + { + lines: this.lines, + version: this.version, + ops: this.ops, + ranges: this.ranges + }, + (e, { doc_id }) => { + this.doc_id = doc_id + return cb(e) + } + ) + }, + + (cb) => { + this.client = RealTimeClient.connect() + return this.client.on('connectionAccepted', cb) + }, + + (cb) => { + return this.client.emit( + 'joinProject', + { project_id: this.project_id }, + cb + ) + }, + + (cb) => { + return this.client.emit( + 'joinDoc', + this.doc_id, + this.fromVersion, + (error, ...rest) => { + ;[...this.returnedArgs] = Array.from(rest) + return cb(error) + } + ) + } + ], + done + ) + }) + + it('should get the doc from the doc updater with the fromVersion', function () { + return MockDocUpdaterServer.getDocument + .calledWith(this.project_id, this.doc_id, this.fromVersion) + .should.equal(true) + }) + + it('should return the doc lines, version, ranges and ops', function () { + return this.returnedArgs.should.deep.equal([ + this.lines, + this.version, + this.ops, + this.ranges + ]) + }) + + return it('should have joined the doc room', function (done) { + return RealTimeClient.getConnectedClient( + this.client.socket.sessionid, + (error, client) => { + expect(Array.from(client.rooms).includes(this.doc_id)).to.equal(true) + return done() + } + ) + }) + }) + + describe('with options', function () { + before(function (done) { + this.options = { encodeRanges: true } + return async.series( + [ + (cb) => { + return FixturesManager.setUpProject( + { + privilegeLevel: 'readAndWrite' + }, + (e, { project_id, user_id }) => { + this.project_id = project_id + this.user_id = user_id + return cb(e) + } + ) + }, + + (cb) => { + return FixturesManager.setUpDoc( + this.project_id, + { + lines: this.lines, + version: this.version, + ops: this.ops, + ranges: this.ranges + }, + (e, { doc_id }) => { + this.doc_id = doc_id + return cb(e) + } + ) + }, + + (cb) => { + this.client = RealTimeClient.connect() + return this.client.on('connectionAccepted', cb) + }, + + (cb) => { + return this.client.emit( + 'joinProject', + { project_id: this.project_id }, + cb + ) + }, + + (cb) => { + return this.client.emit( + 'joinDoc', + this.doc_id, + this.options, + (error, ...rest) => { + ;[...this.returnedArgs] = Array.from(rest) + return cb(error) + } + ) + } + ], + done + ) + }) + + it('should get the doc from the doc updater with the default fromVersion', function () { + return MockDocUpdaterServer.getDocument + .calledWith(this.project_id, this.doc_id, -1) + .should.equal(true) + }) + + it('should return the doc lines, version, ranges and ops', function () { + return this.returnedArgs.should.deep.equal([ + this.lines, + this.version, + this.ops, + this.ranges + ]) + }) + + return it('should have joined the doc room', function (done) { + return RealTimeClient.getConnectedClient( + this.client.socket.sessionid, + (error, client) => { + expect(Array.from(client.rooms).includes(this.doc_id)).to.equal(true) + return done() + } + ) + }) + }) + + return describe('with fromVersion and options', function () { + before(function (done) { + this.fromVersion = 36 + this.options = { encodeRanges: true } + return async.series( + [ + (cb) => { + return FixturesManager.setUpProject( + { + privilegeLevel: 'readAndWrite' + }, + (e, { project_id, user_id }) => { + this.project_id = project_id + this.user_id = user_id + return cb(e) + } + ) + }, + + (cb) => { + return FixturesManager.setUpDoc( + this.project_id, + { + lines: this.lines, + version: this.version, + ops: this.ops, + ranges: this.ranges + }, + (e, { doc_id }) => { + this.doc_id = doc_id + return cb(e) + } + ) + }, + + (cb) => { + this.client = RealTimeClient.connect() + return this.client.on('connectionAccepted', cb) + }, + + (cb) => { + return this.client.emit( + 'joinProject', + { project_id: this.project_id }, + cb + ) + }, + + (cb) => { + return this.client.emit( + 'joinDoc', + this.doc_id, + this.fromVersion, + this.options, + (error, ...rest) => { + ;[...this.returnedArgs] = Array.from(rest) + return cb(error) + } + ) + } + ], + done + ) + }) + + it('should get the doc from the doc updater with the fromVersion', function () { + return MockDocUpdaterServer.getDocument + .calledWith(this.project_id, this.doc_id, this.fromVersion) + .should.equal(true) + }) + + it('should return the doc lines, version, ranges and ops', function () { + return this.returnedArgs.should.deep.equal([ + this.lines, + this.version, + this.ops, + this.ranges + ]) + }) + + return it('should have joined the doc room', function (done) { + return RealTimeClient.getConnectedClient( + this.client.socket.sessionid, + (error, client) => { + expect(Array.from(client.rooms).includes(this.doc_id)).to.equal(true) + return done() + } + ) + }) + }) +}) diff --git a/services/real-time/test/acceptance/js/JoinProjectTests.js b/services/real-time/test/acceptance/js/JoinProjectTests.js index f84af3bba5..051c33d0c7 100644 --- a/services/real-time/test/acceptance/js/JoinProjectTests.js +++ b/services/real-time/test/acceptance/js/JoinProjectTests.js @@ -10,159 +10,199 @@ * DS102: Remove unnecessary code created because of implicit returns * Full docs: https://github.com/decaffeinate/decaffeinate/blob/master/docs/suggestions.md */ -const chai = require("chai"); -const { - expect -} = chai; -chai.should(); +const chai = require('chai') +const { expect } = chai +chai.should() -const RealTimeClient = require("./helpers/RealTimeClient"); -const MockWebServer = require("./helpers/MockWebServer"); -const FixturesManager = require("./helpers/FixturesManager"); +const RealTimeClient = require('./helpers/RealTimeClient') +const MockWebServer = require('./helpers/MockWebServer') +const FixturesManager = require('./helpers/FixturesManager') -const async = require("async"); +const async = require('async') -describe("joinProject", function() { - describe("when authorized", function() { - before(function(done) { - return async.series([ - cb => { - return FixturesManager.setUpProject({ - privilegeLevel: "owner", - project: { - name: "Test Project" - } - }, (e, {project_id, user_id}) => { - this.project_id = project_id; - this.user_id = user_id; - return cb(e); - }); - }, +describe('joinProject', function () { + describe('when authorized', function () { + before(function (done) { + return async.series( + [ + (cb) => { + return FixturesManager.setUpProject( + { + privilegeLevel: 'owner', + project: { + name: 'Test Project' + } + }, + (e, { project_id, user_id }) => { + this.project_id = project_id + this.user_id = user_id + return cb(e) + } + ) + }, - cb => { - this.client = RealTimeClient.connect(); - return this.client.on("connectionAccepted", cb); - }, - - cb => { - return this.client.emit("joinProject", {project_id: this.project_id}, (error, project, privilegeLevel, protocolVersion) => { - this.project = project; - this.privilegeLevel = privilegeLevel; - this.protocolVersion = protocolVersion; - return cb(error); - }); - } - ], done); - }); - - it("should get the project from web", function() { - return MockWebServer.joinProject - .calledWith(this.project_id, this.user_id) - .should.equal(true); - }); - - it("should return the project", function() { - return this.project.should.deep.equal({ - name: "Test Project" - }); - }); - - it("should return the privilege level", function() { - return this.privilegeLevel.should.equal("owner"); - }); - - it("should return the protocolVersion", function() { - return this.protocolVersion.should.equal(2); - }); - - it("should have joined the project room", function(done) { - return RealTimeClient.getConnectedClient(this.client.socket.sessionid, (error, client) => { - expect(Array.from(client.rooms).includes(this.project_id)).to.equal(true); - return done(); - }); - }); - - return it("should have marked the user as connected", function(done) { - return this.client.emit("clientTracking.getConnectedUsers", (error, users) => { - let connected = false; - for (const user of Array.from(users)) { - if ((user.client_id === this.client.publicId) && (user.user_id === this.user_id)) { - connected = true; - break; - } - } - expect(connected).to.equal(true); - return done(); - }); - }); - }); - - describe("when not authorized", function() { - before(function(done) { - return async.series([ - cb => { - return FixturesManager.setUpProject({ - privilegeLevel: null, - project: { - name: "Test Project" - } - }, (e, {project_id, user_id}) => { - this.project_id = project_id; - this.user_id = user_id; - return cb(e); - }); - }, + (cb) => { + this.client = RealTimeClient.connect() + return this.client.on('connectionAccepted', cb) + }, - cb => { - this.client = RealTimeClient.connect(); - return this.client.on("connectionAccepted", cb); - }, - - cb => { - return this.client.emit("joinProject", {project_id: this.project_id}, (error, project, privilegeLevel, protocolVersion) => { - this.error = error; - this.project = project; - this.privilegeLevel = privilegeLevel; - this.protocolVersion = protocolVersion; - return cb(); - }); - } - ], done); - }); - - it("should return an error", function() { - return this.error.message.should.equal("not authorized"); - }); - - return it("should not have joined the project room", function(done) { - return RealTimeClient.getConnectedClient(this.client.socket.sessionid, (error, client) => { - expect(Array.from(client.rooms).includes(this.project_id)).to.equal(false); - return done(); - }); - }); - }); + (cb) => { + return this.client.emit( + 'joinProject', + { project_id: this.project_id }, + (error, project, privilegeLevel, protocolVersion) => { + this.project = project + this.privilegeLevel = privilegeLevel + this.protocolVersion = protocolVersion + return cb(error) + } + ) + } + ], + done + ) + }) - return describe("when over rate limit", function() { - before(function(done) { - return async.series([ - cb => { - this.client = RealTimeClient.connect(); - return this.client.on("connectionAccepted", cb); - }, + it('should get the project from web', function () { + return MockWebServer.joinProject + .calledWith(this.project_id, this.user_id) + .should.equal(true) + }) - cb => { - return this.client.emit("joinProject", {project_id: 'rate-limited'}, error => { - this.error = error; - return cb(); - }); - } - ], done); - }); + it('should return the project', function () { + return this.project.should.deep.equal({ + name: 'Test Project' + }) + }) - return it("should return a TooManyRequests error code", function() { - this.error.message.should.equal("rate-limit hit when joining project"); - return this.error.code.should.equal("TooManyRequests"); - }); - }); -}); + it('should return the privilege level', function () { + return this.privilegeLevel.should.equal('owner') + }) + it('should return the protocolVersion', function () { + return this.protocolVersion.should.equal(2) + }) + + it('should have joined the project room', function (done) { + return RealTimeClient.getConnectedClient( + this.client.socket.sessionid, + (error, client) => { + expect(Array.from(client.rooms).includes(this.project_id)).to.equal( + true + ) + return done() + } + ) + }) + + return it('should have marked the user as connected', function (done) { + return this.client.emit( + 'clientTracking.getConnectedUsers', + (error, users) => { + let connected = false + for (const user of Array.from(users)) { + if ( + user.client_id === this.client.publicId && + user.user_id === this.user_id + ) { + connected = true + break + } + } + expect(connected).to.equal(true) + return done() + } + ) + }) + }) + + describe('when not authorized', function () { + before(function (done) { + return async.series( + [ + (cb) => { + return FixturesManager.setUpProject( + { + privilegeLevel: null, + project: { + name: 'Test Project' + } + }, + (e, { project_id, user_id }) => { + this.project_id = project_id + this.user_id = user_id + return cb(e) + } + ) + }, + + (cb) => { + this.client = RealTimeClient.connect() + return this.client.on('connectionAccepted', cb) + }, + + (cb) => { + return this.client.emit( + 'joinProject', + { project_id: this.project_id }, + (error, project, privilegeLevel, protocolVersion) => { + this.error = error + this.project = project + this.privilegeLevel = privilegeLevel + this.protocolVersion = protocolVersion + return cb() + } + ) + } + ], + done + ) + }) + + it('should return an error', function () { + return this.error.message.should.equal('not authorized') + }) + + return it('should not have joined the project room', function (done) { + return RealTimeClient.getConnectedClient( + this.client.socket.sessionid, + (error, client) => { + expect(Array.from(client.rooms).includes(this.project_id)).to.equal( + false + ) + return done() + } + ) + }) + }) + + return describe('when over rate limit', function () { + before(function (done) { + return async.series( + [ + (cb) => { + this.client = RealTimeClient.connect() + return this.client.on('connectionAccepted', cb) + }, + + (cb) => { + return this.client.emit( + 'joinProject', + { project_id: 'rate-limited' }, + (error) => { + this.error = error + return cb() + } + ) + } + ], + done + ) + }) + + return it('should return a TooManyRequests error code', function () { + this.error.message.should.equal('rate-limit hit when joining project') + return this.error.code.should.equal('TooManyRequests') + }) + }) +}) diff --git a/services/real-time/test/acceptance/js/LeaveDocTests.js b/services/real-time/test/acceptance/js/LeaveDocTests.js index 3f396e3df5..a842087522 100644 --- a/services/real-time/test/acceptance/js/LeaveDocTests.js +++ b/services/real-time/test/acceptance/js/LeaveDocTests.js @@ -13,117 +13,164 @@ * DS207: Consider shorter variations of null checks * Full docs: https://github.com/decaffeinate/decaffeinate/blob/master/docs/suggestions.md */ -const chai = require("chai"); -const { - expect -} = chai; -chai.should(); -const sinon = require("sinon"); +const chai = require('chai') +const { expect } = chai +chai.should() +const sinon = require('sinon') -const RealTimeClient = require("./helpers/RealTimeClient"); -const MockDocUpdaterServer = require("./helpers/MockDocUpdaterServer"); -const FixturesManager = require("./helpers/FixturesManager"); -const logger = require("logger-sharelatex"); +const RealTimeClient = require('./helpers/RealTimeClient') +const MockDocUpdaterServer = require('./helpers/MockDocUpdaterServer') +const FixturesManager = require('./helpers/FixturesManager') +const logger = require('logger-sharelatex') -const async = require("async"); +const async = require('async') -describe("leaveDoc", function() { - before(function() { - this.lines = ["test", "doc", "lines"]; - this.version = 42; - this.ops = ["mock", "doc", "ops"]; - sinon.spy(logger, "error"); - sinon.spy(logger, "warn"); - sinon.spy(logger, "log"); - return this.other_doc_id = FixturesManager.getRandomId(); - }); - - after(function() { - logger.error.restore(); // remove the spy - logger.warn.restore(); - return logger.log.restore(); - }); +describe('leaveDoc', function () { + before(function () { + this.lines = ['test', 'doc', 'lines'] + this.version = 42 + this.ops = ['mock', 'doc', 'ops'] + sinon.spy(logger, 'error') + sinon.spy(logger, 'warn') + sinon.spy(logger, 'log') + return (this.other_doc_id = FixturesManager.getRandomId()) + }) - return describe("when joined to a doc", function() { - beforeEach(function(done) { - return async.series([ - cb => { - return FixturesManager.setUpProject({ - privilegeLevel: "readAndWrite" - }, (e, {project_id, user_id}) => { - this.project_id = project_id; - this.user_id = user_id; - return cb(e); - }); - }, - - cb => { - return FixturesManager.setUpDoc(this.project_id, {lines: this.lines, version: this.version, ops: this.ops}, (e, {doc_id}) => { - this.doc_id = doc_id; - return cb(e); - }); - }, - - cb => { - this.client = RealTimeClient.connect(); - return this.client.on("connectionAccepted", cb); - }, - - cb => { - return this.client.emit("joinProject", {project_id: this.project_id}, cb); - }, - - cb => { - return this.client.emit("joinDoc", this.doc_id, (error, ...rest) => { [...this.returnedArgs] = Array.from(rest); return cb(error); }); - } - ], done); - }); - - describe("then leaving the doc", function() { - beforeEach(function(done) { - return this.client.emit("leaveDoc", this.doc_id, (error) => { - if (error != null) { throw error; } - return done(); - }); - }); - - return it("should have left the doc room", function(done) { - return RealTimeClient.getConnectedClient(this.client.socket.sessionid, (error, client) => { - expect(Array.from(client.rooms).includes(this.doc_id)).to.equal(false); - return done(); - }); - }); - }); + after(function () { + logger.error.restore() // remove the spy + logger.warn.restore() + return logger.log.restore() + }) - describe("when sending a leaveDoc request before the previous joinDoc request has completed", function() { - beforeEach(function(done) { - this.client.emit("leaveDoc", this.doc_id, () => {}); - this.client.emit("joinDoc", this.doc_id, () => {}); - return this.client.emit("leaveDoc", this.doc_id, (error) => { - if (error != null) { throw error; } - return done(); - }); - }); + return describe('when joined to a doc', function () { + beforeEach(function (done) { + return async.series( + [ + (cb) => { + return FixturesManager.setUpProject( + { + privilegeLevel: 'readAndWrite' + }, + (e, { project_id, user_id }) => { + this.project_id = project_id + this.user_id = user_id + return cb(e) + } + ) + }, - it("should not trigger an error", function() { return sinon.assert.neverCalledWith(logger.error, sinon.match.any, "not subscribed - shouldn't happen"); }); + (cb) => { + return FixturesManager.setUpDoc( + this.project_id, + { lines: this.lines, version: this.version, ops: this.ops }, + (e, { doc_id }) => { + this.doc_id = doc_id + return cb(e) + } + ) + }, - return it("should have left the doc room", function(done) { - return RealTimeClient.getConnectedClient(this.client.socket.sessionid, (error, client) => { - expect(Array.from(client.rooms).includes(this.doc_id)).to.equal(false); - return done(); - }); - }); - }); + (cb) => { + this.client = RealTimeClient.connect() + return this.client.on('connectionAccepted', cb) + }, - return describe("when sending a leaveDoc for a room the client has not joined ", function() { - beforeEach(function(done) { - return this.client.emit("leaveDoc", this.other_doc_id, (error) => { - if (error != null) { throw error; } - return done(); - }); - }); + (cb) => { + return this.client.emit( + 'joinProject', + { project_id: this.project_id }, + cb + ) + }, - return it("should trigger a low level message only", function() { return sinon.assert.calledWith(logger.log, sinon.match.any, "ignoring request from client to leave room it is not in"); }); - }); - }); -}); + (cb) => { + return this.client.emit( + 'joinDoc', + this.doc_id, + (error, ...rest) => { + ;[...this.returnedArgs] = Array.from(rest) + return cb(error) + } + ) + } + ], + done + ) + }) + + describe('then leaving the doc', function () { + beforeEach(function (done) { + return this.client.emit('leaveDoc', this.doc_id, (error) => { + if (error != null) { + throw error + } + return done() + }) + }) + + return it('should have left the doc room', function (done) { + return RealTimeClient.getConnectedClient( + this.client.socket.sessionid, + (error, client) => { + expect(Array.from(client.rooms).includes(this.doc_id)).to.equal( + false + ) + return done() + } + ) + }) + }) + + describe('when sending a leaveDoc request before the previous joinDoc request has completed', function () { + beforeEach(function (done) { + this.client.emit('leaveDoc', this.doc_id, () => {}) + this.client.emit('joinDoc', this.doc_id, () => {}) + return this.client.emit('leaveDoc', this.doc_id, (error) => { + if (error != null) { + throw error + } + return done() + }) + }) + + it('should not trigger an error', function () { + return sinon.assert.neverCalledWith( + logger.error, + sinon.match.any, + "not subscribed - shouldn't happen" + ) + }) + + return it('should have left the doc room', function (done) { + return RealTimeClient.getConnectedClient( + this.client.socket.sessionid, + (error, client) => { + expect(Array.from(client.rooms).includes(this.doc_id)).to.equal( + false + ) + return done() + } + ) + }) + }) + + return describe('when sending a leaveDoc for a room the client has not joined ', function () { + beforeEach(function (done) { + return this.client.emit('leaveDoc', this.other_doc_id, (error) => { + if (error != null) { + throw error + } + return done() + }) + }) + + return it('should trigger a low level message only', function () { + return sinon.assert.calledWith( + logger.log, + sinon.match.any, + 'ignoring request from client to leave room it is not in' + ) + }) + }) + }) +}) diff --git a/services/real-time/test/acceptance/js/LeaveProjectTests.js b/services/real-time/test/acceptance/js/LeaveProjectTests.js index 36a17fe081..61976d481f 100644 --- a/services/real-time/test/acceptance/js/LeaveProjectTests.js +++ b/services/real-time/test/acceptance/js/LeaveProjectTests.js @@ -11,203 +11,260 @@ * DS102: Remove unnecessary code created because of implicit returns * Full docs: https://github.com/decaffeinate/decaffeinate/blob/master/docs/suggestions.md */ -const RealTimeClient = require("./helpers/RealTimeClient"); -const MockDocUpdaterServer = require("./helpers/MockDocUpdaterServer"); -const FixturesManager = require("./helpers/FixturesManager"); +const RealTimeClient = require('./helpers/RealTimeClient') +const MockDocUpdaterServer = require('./helpers/MockDocUpdaterServer') +const FixturesManager = require('./helpers/FixturesManager') -const async = require("async"); +const async = require('async') -const settings = require("settings-sharelatex"); -const redis = require("redis-sharelatex"); -const rclient = redis.createClient(settings.redis.pubsub); +const settings = require('settings-sharelatex') +const redis = require('redis-sharelatex') +const rclient = redis.createClient(settings.redis.pubsub) -describe("leaveProject", function() { - before(function(done) { return MockDocUpdaterServer.run(done); }); +describe('leaveProject', function () { + before(function (done) { + return MockDocUpdaterServer.run(done) + }) - describe("with other clients in the project", function() { - before(function(done) { - return async.series([ - cb => { - return FixturesManager.setUpProject({ - privilegeLevel: "owner", - project: { - name: "Test Project" - } - }, (e, {project_id, user_id}) => { this.project_id = project_id; this.user_id = user_id; return cb(); }); - }, + describe('with other clients in the project', function () { + before(function (done) { + return async.series( + [ + (cb) => { + return FixturesManager.setUpProject( + { + privilegeLevel: 'owner', + project: { + name: 'Test Project' + } + }, + (e, { project_id, user_id }) => { + this.project_id = project_id + this.user_id = user_id + return cb() + } + ) + }, - cb => { - this.clientA = RealTimeClient.connect(); - return this.clientA.on("connectionAccepted", cb); - }, + (cb) => { + this.clientA = RealTimeClient.connect() + return this.clientA.on('connectionAccepted', cb) + }, - cb => { - this.clientB = RealTimeClient.connect(); - this.clientB.on("connectionAccepted", cb); + (cb) => { + this.clientB = RealTimeClient.connect() + this.clientB.on('connectionAccepted', cb) - this.clientBDisconnectMessages = []; - return this.clientB.on("clientTracking.clientDisconnected", data => { - return this.clientBDisconnectMessages.push(data); - }); - }, + this.clientBDisconnectMessages = [] + return this.clientB.on( + 'clientTracking.clientDisconnected', + (data) => { + return this.clientBDisconnectMessages.push(data) + } + ) + }, - cb => { - return this.clientA.emit("joinProject", {project_id: this.project_id}, (error, project, privilegeLevel, protocolVersion) => { - this.project = project; - this.privilegeLevel = privilegeLevel; - this.protocolVersion = protocolVersion; - return cb(error); - }); - }, + (cb) => { + return this.clientA.emit( + 'joinProject', + { project_id: this.project_id }, + (error, project, privilegeLevel, protocolVersion) => { + this.project = project + this.privilegeLevel = privilegeLevel + this.protocolVersion = protocolVersion + return cb(error) + } + ) + }, - cb => { - return this.clientB.emit("joinProject", {project_id: this.project_id}, (error, project, privilegeLevel, protocolVersion) => { - this.project = project; - this.privilegeLevel = privilegeLevel; - this.protocolVersion = protocolVersion; - return cb(error); - }); - }, + (cb) => { + return this.clientB.emit( + 'joinProject', + { project_id: this.project_id }, + (error, project, privilegeLevel, protocolVersion) => { + this.project = project + this.privilegeLevel = privilegeLevel + this.protocolVersion = protocolVersion + return cb(error) + } + ) + }, - cb => { - return FixturesManager.setUpDoc(this.project_id, {lines: this.lines, version: this.version, ops: this.ops}, (e, {doc_id}) => { - this.doc_id = doc_id; - return cb(e); - }); - }, + (cb) => { + return FixturesManager.setUpDoc( + this.project_id, + { lines: this.lines, version: this.version, ops: this.ops }, + (e, { doc_id }) => { + this.doc_id = doc_id + return cb(e) + } + ) + }, - cb => { - return this.clientA.emit("joinDoc", this.doc_id, cb); - }, - cb => { - return this.clientB.emit("joinDoc", this.doc_id, cb); - }, + (cb) => { + return this.clientA.emit('joinDoc', this.doc_id, cb) + }, + (cb) => { + return this.clientB.emit('joinDoc', this.doc_id, cb) + }, - cb => { - // leaveProject is called when the client disconnects - this.clientA.on("disconnect", () => cb()); - return this.clientA.disconnect(); - }, + (cb) => { + // leaveProject is called when the client disconnects + this.clientA.on('disconnect', () => cb()) + return this.clientA.disconnect() + }, - cb => { - // The API waits a little while before flushing changes - return setTimeout(done, 1000); - } + (cb) => { + // The API waits a little while before flushing changes + return setTimeout(done, 1000) + } + ], + done + ) + }) - ], done); - }); + it('should emit a disconnect message to the room', function () { + return this.clientBDisconnectMessages.should.deep.equal([ + this.clientA.publicId + ]) + }) - it("should emit a disconnect message to the room", function() { - return this.clientBDisconnectMessages.should.deep.equal([this.clientA.publicId]); - }); + it('should no longer list the client in connected users', function (done) { + return this.clientB.emit( + 'clientTracking.getConnectedUsers', + (error, users) => { + for (const user of Array.from(users)) { + if (user.client_id === this.clientA.publicId) { + throw 'Expected clientA to not be listed in connected users' + } + } + return done() + } + ) + }) - it("should no longer list the client in connected users", function(done) { - return this.clientB.emit("clientTracking.getConnectedUsers", (error, users) => { - for (const user of Array.from(users)) { - if (user.client_id === this.clientA.publicId) { - throw "Expected clientA to not be listed in connected users"; - } - } - return done(); - }); - }); + it('should not flush the project to the document updater', function () { + return MockDocUpdaterServer.deleteProject + .calledWith(this.project_id) + .should.equal(false) + }) - it("should not flush the project to the document updater", function() { - return MockDocUpdaterServer.deleteProject - .calledWith(this.project_id) - .should.equal(false); - }); + it('should remain subscribed to the editor-events channels', function (done) { + rclient.pubsub('CHANNELS', (err, resp) => { + if (err) { + return done(err) + } + resp.should.include(`editor-events:${this.project_id}`) + return done() + }) + return null + }) - it("should remain subscribed to the editor-events channels", function(done) { - rclient.pubsub('CHANNELS', (err, resp) => { - if (err) { return done(err); } - resp.should.include(`editor-events:${this.project_id}`); - return done(); - }); - return null; - }); + return it('should remain subscribed to the applied-ops channels', function (done) { + rclient.pubsub('CHANNELS', (err, resp) => { + if (err) { + return done(err) + } + resp.should.include(`applied-ops:${this.doc_id}`) + return done() + }) + return null + }) + }) - return it("should remain subscribed to the applied-ops channels", function(done) { - rclient.pubsub('CHANNELS', (err, resp) => { - if (err) { return done(err); } - resp.should.include(`applied-ops:${this.doc_id}`); - return done(); - }); - return null; - }); - }); + return describe('with no other clients in the project', function () { + before(function (done) { + return async.series( + [ + (cb) => { + return FixturesManager.setUpProject( + { + privilegeLevel: 'owner', + project: { + name: 'Test Project' + } + }, + (e, { project_id, user_id }) => { + this.project_id = project_id + this.user_id = user_id + return cb() + } + ) + }, - return describe("with no other clients in the project", function() { - before(function(done) { - return async.series([ - cb => { - return FixturesManager.setUpProject({ - privilegeLevel: "owner", - project: { - name: "Test Project" - } - }, (e, {project_id, user_id}) => { this.project_id = project_id; this.user_id = user_id; return cb(); }); - }, + (cb) => { + this.clientA = RealTimeClient.connect() + return this.clientA.on('connect', cb) + }, - cb => { - this.clientA = RealTimeClient.connect(); - return this.clientA.on("connect", cb); - }, + (cb) => { + return this.clientA.emit( + 'joinProject', + { project_id: this.project_id }, + (error, project, privilegeLevel, protocolVersion) => { + this.project = project + this.privilegeLevel = privilegeLevel + this.protocolVersion = protocolVersion + return cb(error) + } + ) + }, - cb => { - return this.clientA.emit("joinProject", {project_id: this.project_id}, (error, project, privilegeLevel, protocolVersion) => { - this.project = project; - this.privilegeLevel = privilegeLevel; - this.protocolVersion = protocolVersion; - return cb(error); - }); - }, + (cb) => { + return FixturesManager.setUpDoc( + this.project_id, + { lines: this.lines, version: this.version, ops: this.ops }, + (e, { doc_id }) => { + this.doc_id = doc_id + return cb(e) + } + ) + }, + (cb) => { + return this.clientA.emit('joinDoc', this.doc_id, cb) + }, - cb => { - return FixturesManager.setUpDoc(this.project_id, {lines: this.lines, version: this.version, ops: this.ops}, (e, {doc_id}) => { - this.doc_id = doc_id; - return cb(e); - }); - }, - cb => { - return this.clientA.emit("joinDoc", this.doc_id, cb); - }, + (cb) => { + // leaveProject is called when the client disconnects + this.clientA.on('disconnect', () => cb()) + return this.clientA.disconnect() + }, - cb => { - // leaveProject is called when the client disconnects - this.clientA.on("disconnect", () => cb()); - return this.clientA.disconnect(); - }, + (cb) => { + // The API waits a little while before flushing changes + return setTimeout(done, 1000) + } + ], + done + ) + }) - cb => { - // The API waits a little while before flushing changes - return setTimeout(done, 1000); - } - ], done); - }); + it('should flush the project to the document updater', function () { + return MockDocUpdaterServer.deleteProject + .calledWith(this.project_id) + .should.equal(true) + }) - it("should flush the project to the document updater", function() { - return MockDocUpdaterServer.deleteProject - .calledWith(this.project_id) - .should.equal(true); - }); + it('should not subscribe to the editor-events channels anymore', function (done) { + rclient.pubsub('CHANNELS', (err, resp) => { + if (err) { + return done(err) + } + resp.should.not.include(`editor-events:${this.project_id}`) + return done() + }) + return null + }) - it("should not subscribe to the editor-events channels anymore", function(done) { - rclient.pubsub('CHANNELS', (err, resp) => { - if (err) { return done(err); } - resp.should.not.include(`editor-events:${this.project_id}`); - return done(); - }); - return null; - }); - - return it("should not subscribe to the applied-ops channels anymore", function(done) { - rclient.pubsub('CHANNELS', (err, resp) => { - if (err) { return done(err); } - resp.should.not.include(`applied-ops:${this.doc_id}`); - return done(); - }); - return null; - }); - }); -}); + return it('should not subscribe to the applied-ops channels anymore', function (done) { + rclient.pubsub('CHANNELS', (err, resp) => { + if (err) { + return done(err) + } + resp.should.not.include(`applied-ops:${this.doc_id}`) + return done() + }) + return null + }) + }) +}) diff --git a/services/real-time/test/acceptance/js/PubSubRace.js b/services/real-time/test/acceptance/js/PubSubRace.js index 34d526d820..a824ef3e82 100644 --- a/services/real-time/test/acceptance/js/PubSubRace.js +++ b/services/real-time/test/acceptance/js/PubSubRace.js @@ -9,275 +9,365 @@ * DS102: Remove unnecessary code created because of implicit returns * Full docs: https://github.com/decaffeinate/decaffeinate/blob/master/docs/suggestions.md */ -const RealTimeClient = require("./helpers/RealTimeClient"); -const MockDocUpdaterServer = require("./helpers/MockDocUpdaterServer"); -const FixturesManager = require("./helpers/FixturesManager"); +const RealTimeClient = require('./helpers/RealTimeClient') +const MockDocUpdaterServer = require('./helpers/MockDocUpdaterServer') +const FixturesManager = require('./helpers/FixturesManager') -const async = require("async"); +const async = require('async') -const settings = require("settings-sharelatex"); -const redis = require("redis-sharelatex"); -const rclient = redis.createClient(settings.redis.pubsub); +const settings = require('settings-sharelatex') +const redis = require('redis-sharelatex') +const rclient = redis.createClient(settings.redis.pubsub) -describe("PubSubRace", function() { - before(function(done) { return MockDocUpdaterServer.run(done); }); +describe('PubSubRace', function () { + before(function (done) { + return MockDocUpdaterServer.run(done) + }) - describe("when the client leaves a doc before joinDoc completes", function() { - before(function(done) { - return async.series([ - cb => { - return FixturesManager.setUpProject({ - privilegeLevel: "owner", - project: { - name: "Test Project" - } - }, (e, {project_id, user_id}) => { this.project_id = project_id; this.user_id = user_id; return cb(); }); - }, + describe('when the client leaves a doc before joinDoc completes', function () { + before(function (done) { + return async.series( + [ + (cb) => { + return FixturesManager.setUpProject( + { + privilegeLevel: 'owner', + project: { + name: 'Test Project' + } + }, + (e, { project_id, user_id }) => { + this.project_id = project_id + this.user_id = user_id + return cb() + } + ) + }, - cb => { - this.clientA = RealTimeClient.connect(); - return this.clientA.on("connect", cb); - }, + (cb) => { + this.clientA = RealTimeClient.connect() + return this.clientA.on('connect', cb) + }, - cb => { - return this.clientA.emit("joinProject", {project_id: this.project_id}, (error, project, privilegeLevel, protocolVersion) => { - this.project = project; - this.privilegeLevel = privilegeLevel; - this.protocolVersion = protocolVersion; - return cb(error); - }); - }, + (cb) => { + return this.clientA.emit( + 'joinProject', + { project_id: this.project_id }, + (error, project, privilegeLevel, protocolVersion) => { + this.project = project + this.privilegeLevel = privilegeLevel + this.protocolVersion = protocolVersion + return cb(error) + } + ) + }, - cb => { - return FixturesManager.setUpDoc(this.project_id, {lines: this.lines, version: this.version, ops: this.ops}, (e, {doc_id}) => { - this.doc_id = doc_id; - return cb(e); - }); - }, + (cb) => { + return FixturesManager.setUpDoc( + this.project_id, + { lines: this.lines, version: this.version, ops: this.ops }, + (e, { doc_id }) => { + this.doc_id = doc_id + return cb(e) + } + ) + }, - cb => { - this.clientA.emit("joinDoc", this.doc_id, () => {}); - // leave before joinDoc completes - return this.clientA.emit("leaveDoc", this.doc_id, cb); - }, + (cb) => { + this.clientA.emit('joinDoc', this.doc_id, () => {}) + // leave before joinDoc completes + return this.clientA.emit('leaveDoc', this.doc_id, cb) + }, - cb => { - // wait for subscribe and unsubscribe - return setTimeout(cb, 100); - } - ], done); - }); + (cb) => { + // wait for subscribe and unsubscribe + return setTimeout(cb, 100) + } + ], + done + ) + }) - return it("should not subscribe to the applied-ops channels anymore", function(done) { - rclient.pubsub('CHANNELS', (err, resp) => { - if (err) { return done(err); } - resp.should.not.include(`applied-ops:${this.doc_id}`); - return done(); - }); - return null; - }); - }); + return it('should not subscribe to the applied-ops channels anymore', function (done) { + rclient.pubsub('CHANNELS', (err, resp) => { + if (err) { + return done(err) + } + resp.should.not.include(`applied-ops:${this.doc_id}`) + return done() + }) + return null + }) + }) - describe("when the client emits joinDoc and leaveDoc requests frequently and leaves eventually", function() { - before(function(done) { - return async.series([ - cb => { - return FixturesManager.setUpProject({ - privilegeLevel: "owner", - project: { - name: "Test Project" - } - }, (e, {project_id, user_id}) => { this.project_id = project_id; this.user_id = user_id; return cb(); }); - }, + describe('when the client emits joinDoc and leaveDoc requests frequently and leaves eventually', function () { + before(function (done) { + return async.series( + [ + (cb) => { + return FixturesManager.setUpProject( + { + privilegeLevel: 'owner', + project: { + name: 'Test Project' + } + }, + (e, { project_id, user_id }) => { + this.project_id = project_id + this.user_id = user_id + return cb() + } + ) + }, - cb => { - this.clientA = RealTimeClient.connect(); - return this.clientA.on("connect", cb); - }, + (cb) => { + this.clientA = RealTimeClient.connect() + return this.clientA.on('connect', cb) + }, - cb => { - return this.clientA.emit("joinProject", {project_id: this.project_id}, (error, project, privilegeLevel, protocolVersion) => { - this.project = project; - this.privilegeLevel = privilegeLevel; - this.protocolVersion = protocolVersion; - return cb(error); - }); - }, + (cb) => { + return this.clientA.emit( + 'joinProject', + { project_id: this.project_id }, + (error, project, privilegeLevel, protocolVersion) => { + this.project = project + this.privilegeLevel = privilegeLevel + this.protocolVersion = protocolVersion + return cb(error) + } + ) + }, - cb => { - return FixturesManager.setUpDoc(this.project_id, {lines: this.lines, version: this.version, ops: this.ops}, (e, {doc_id}) => { - this.doc_id = doc_id; - return cb(e); - }); - }, + (cb) => { + return FixturesManager.setUpDoc( + this.project_id, + { lines: this.lines, version: this.version, ops: this.ops }, + (e, { doc_id }) => { + this.doc_id = doc_id + return cb(e) + } + ) + }, - cb => { - this.clientA.emit("joinDoc", this.doc_id, () => {}); - this.clientA.emit("leaveDoc", this.doc_id, () => {}); - this.clientA.emit("joinDoc", this.doc_id, () => {}); - this.clientA.emit("leaveDoc", this.doc_id, () => {}); - this.clientA.emit("joinDoc", this.doc_id, () => {}); - this.clientA.emit("leaveDoc", this.doc_id, () => {}); - this.clientA.emit("joinDoc", this.doc_id, () => {}); - this.clientA.emit("leaveDoc", this.doc_id, () => {}); - this.clientA.emit("joinDoc", this.doc_id, () => {}); - return this.clientA.emit("leaveDoc", this.doc_id, cb); - }, + (cb) => { + this.clientA.emit('joinDoc', this.doc_id, () => {}) + this.clientA.emit('leaveDoc', this.doc_id, () => {}) + this.clientA.emit('joinDoc', this.doc_id, () => {}) + this.clientA.emit('leaveDoc', this.doc_id, () => {}) + this.clientA.emit('joinDoc', this.doc_id, () => {}) + this.clientA.emit('leaveDoc', this.doc_id, () => {}) + this.clientA.emit('joinDoc', this.doc_id, () => {}) + this.clientA.emit('leaveDoc', this.doc_id, () => {}) + this.clientA.emit('joinDoc', this.doc_id, () => {}) + return this.clientA.emit('leaveDoc', this.doc_id, cb) + }, - cb => { - // wait for subscribe and unsubscribe - return setTimeout(cb, 100); - } - ], done); - }); + (cb) => { + // wait for subscribe and unsubscribe + return setTimeout(cb, 100) + } + ], + done + ) + }) - return it("should not subscribe to the applied-ops channels anymore", function(done) { - rclient.pubsub('CHANNELS', (err, resp) => { - if (err) { return done(err); } - resp.should.not.include(`applied-ops:${this.doc_id}`); - return done(); - }); - return null; - }); - }); + return it('should not subscribe to the applied-ops channels anymore', function (done) { + rclient.pubsub('CHANNELS', (err, resp) => { + if (err) { + return done(err) + } + resp.should.not.include(`applied-ops:${this.doc_id}`) + return done() + }) + return null + }) + }) - describe("when the client emits joinDoc and leaveDoc requests frequently and remains in the doc", function() { - before(function(done) { - return async.series([ - cb => { - return FixturesManager.setUpProject({ - privilegeLevel: "owner", - project: { - name: "Test Project" - } - }, (e, {project_id, user_id}) => { this.project_id = project_id; this.user_id = user_id; return cb(); }); - }, + describe('when the client emits joinDoc and leaveDoc requests frequently and remains in the doc', function () { + before(function (done) { + return async.series( + [ + (cb) => { + return FixturesManager.setUpProject( + { + privilegeLevel: 'owner', + project: { + name: 'Test Project' + } + }, + (e, { project_id, user_id }) => { + this.project_id = project_id + this.user_id = user_id + return cb() + } + ) + }, - cb => { - this.clientA = RealTimeClient.connect(); - return this.clientA.on("connect", cb); - }, + (cb) => { + this.clientA = RealTimeClient.connect() + return this.clientA.on('connect', cb) + }, - cb => { - return this.clientA.emit("joinProject", {project_id: this.project_id}, (error, project, privilegeLevel, protocolVersion) => { - this.project = project; - this.privilegeLevel = privilegeLevel; - this.protocolVersion = protocolVersion; - return cb(error); - }); - }, + (cb) => { + return this.clientA.emit( + 'joinProject', + { project_id: this.project_id }, + (error, project, privilegeLevel, protocolVersion) => { + this.project = project + this.privilegeLevel = privilegeLevel + this.protocolVersion = protocolVersion + return cb(error) + } + ) + }, - cb => { - return FixturesManager.setUpDoc(this.project_id, {lines: this.lines, version: this.version, ops: this.ops}, (e, {doc_id}) => { - this.doc_id = doc_id; - return cb(e); - }); - }, + (cb) => { + return FixturesManager.setUpDoc( + this.project_id, + { lines: this.lines, version: this.version, ops: this.ops }, + (e, { doc_id }) => { + this.doc_id = doc_id + return cb(e) + } + ) + }, - cb => { - this.clientA.emit("joinDoc", this.doc_id, () => {}); - this.clientA.emit("leaveDoc", this.doc_id, () => {}); - this.clientA.emit("joinDoc", this.doc_id, () => {}); - this.clientA.emit("leaveDoc", this.doc_id, () => {}); - this.clientA.emit("joinDoc", this.doc_id, () => {}); - this.clientA.emit("leaveDoc", this.doc_id, () => {}); - this.clientA.emit("joinDoc", this.doc_id, () => {}); - this.clientA.emit("leaveDoc", this.doc_id, () => {}); - return this.clientA.emit("joinDoc", this.doc_id, cb); - }, + (cb) => { + this.clientA.emit('joinDoc', this.doc_id, () => {}) + this.clientA.emit('leaveDoc', this.doc_id, () => {}) + this.clientA.emit('joinDoc', this.doc_id, () => {}) + this.clientA.emit('leaveDoc', this.doc_id, () => {}) + this.clientA.emit('joinDoc', this.doc_id, () => {}) + this.clientA.emit('leaveDoc', this.doc_id, () => {}) + this.clientA.emit('joinDoc', this.doc_id, () => {}) + this.clientA.emit('leaveDoc', this.doc_id, () => {}) + return this.clientA.emit('joinDoc', this.doc_id, cb) + }, - cb => { - // wait for subscribe and unsubscribe - return setTimeout(cb, 100); - } - ], done); - }); + (cb) => { + // wait for subscribe and unsubscribe + return setTimeout(cb, 100) + } + ], + done + ) + }) - return it("should subscribe to the applied-ops channels", function(done) { - rclient.pubsub('CHANNELS', (err, resp) => { - if (err) { return done(err); } - resp.should.include(`applied-ops:${this.doc_id}`); - return done(); - }); - return null; - }); - }); + return it('should subscribe to the applied-ops channels', function (done) { + rclient.pubsub('CHANNELS', (err, resp) => { + if (err) { + return done(err) + } + resp.should.include(`applied-ops:${this.doc_id}`) + return done() + }) + return null + }) + }) - return describe("when the client disconnects before joinDoc completes", function() { - before(function(done) { - return async.series([ - cb => { - return FixturesManager.setUpProject({ - privilegeLevel: "owner", - project: { - name: "Test Project" - } - }, (e, {project_id, user_id}) => { this.project_id = project_id; this.user_id = user_id; return cb(); }); - }, + return describe('when the client disconnects before joinDoc completes', function () { + before(function (done) { + return async.series( + [ + (cb) => { + return FixturesManager.setUpProject( + { + privilegeLevel: 'owner', + project: { + name: 'Test Project' + } + }, + (e, { project_id, user_id }) => { + this.project_id = project_id + this.user_id = user_id + return cb() + } + ) + }, - cb => { - this.clientA = RealTimeClient.connect(); - return this.clientA.on("connect", cb); - }, + (cb) => { + this.clientA = RealTimeClient.connect() + return this.clientA.on('connect', cb) + }, - cb => { - return this.clientA.emit("joinProject", {project_id: this.project_id}, (error, project, privilegeLevel, protocolVersion) => { - this.project = project; - this.privilegeLevel = privilegeLevel; - this.protocolVersion = protocolVersion; - return cb(error); - }); - }, + (cb) => { + return this.clientA.emit( + 'joinProject', + { project_id: this.project_id }, + (error, project, privilegeLevel, protocolVersion) => { + this.project = project + this.privilegeLevel = privilegeLevel + this.protocolVersion = protocolVersion + return cb(error) + } + ) + }, - cb => { - return FixturesManager.setUpDoc(this.project_id, {lines: this.lines, version: this.version, ops: this.ops}, (e, {doc_id}) => { - this.doc_id = doc_id; - return cb(e); - }); - }, + (cb) => { + return FixturesManager.setUpDoc( + this.project_id, + { lines: this.lines, version: this.version, ops: this.ops }, + (e, { doc_id }) => { + this.doc_id = doc_id + return cb(e) + } + ) + }, - cb => { - let joinDocCompleted = false; - this.clientA.emit("joinDoc", this.doc_id, () => joinDocCompleted = true); - // leave before joinDoc completes - return setTimeout(() => { - if (joinDocCompleted) { - return cb(new Error('joinDocCompleted -- lower timeout')); - } - this.clientA.on("disconnect", () => cb()); - return this.clientA.disconnect(); - } - // socket.io processes joinDoc and disconnect with different delays: - // - joinDoc goes through two process.nextTick - // - disconnect goes through one process.nextTick - // We have to inject the disconnect event into a different event loop - // cycle. - , 3); - }, + (cb) => { + let joinDocCompleted = false + this.clientA.emit( + 'joinDoc', + this.doc_id, + () => (joinDocCompleted = true) + ) + // leave before joinDoc completes + return setTimeout( + () => { + if (joinDocCompleted) { + return cb(new Error('joinDocCompleted -- lower timeout')) + } + this.clientA.on('disconnect', () => cb()) + return this.clientA.disconnect() + }, + // socket.io processes joinDoc and disconnect with different delays: + // - joinDoc goes through two process.nextTick + // - disconnect goes through one process.nextTick + // We have to inject the disconnect event into a different event loop + // cycle. + 3 + ) + }, - cb => { - // wait for subscribe and unsubscribe - return setTimeout(cb, 100); - } - ], done); - }); + (cb) => { + // wait for subscribe and unsubscribe + return setTimeout(cb, 100) + } + ], + done + ) + }) - it("should not subscribe to the editor-events channels anymore", function(done) { - rclient.pubsub('CHANNELS', (err, resp) => { - if (err) { return done(err); } - resp.should.not.include(`editor-events:${this.project_id}`); - return done(); - }); - return null; - }); + it('should not subscribe to the editor-events channels anymore', function (done) { + rclient.pubsub('CHANNELS', (err, resp) => { + if (err) { + return done(err) + } + resp.should.not.include(`editor-events:${this.project_id}`) + return done() + }) + return null + }) - return it("should not subscribe to the applied-ops channels anymore", function(done) { - rclient.pubsub('CHANNELS', (err, resp) => { - if (err) { return done(err); } - resp.should.not.include(`applied-ops:${this.doc_id}`); - return done(); - }); - return null; - }); - }); -}); + return it('should not subscribe to the applied-ops channels anymore', function (done) { + rclient.pubsub('CHANNELS', (err, resp) => { + if (err) { + return done(err) + } + resp.should.not.include(`applied-ops:${this.doc_id}`) + return done() + }) + return null + }) + }) +}) diff --git a/services/real-time/test/acceptance/js/ReceiveUpdateTests.js b/services/real-time/test/acceptance/js/ReceiveUpdateTests.js index d141b27745..9c65be19f9 100644 --- a/services/real-time/test/acceptance/js/ReceiveUpdateTests.js +++ b/services/real-time/test/acceptance/js/ReceiveUpdateTests.js @@ -11,271 +11,339 @@ * DS207: Consider shorter variations of null checks * Full docs: https://github.com/decaffeinate/decaffeinate/blob/master/docs/suggestions.md */ -const chai = require("chai"); -const { - expect -} = chai; -chai.should(); +const chai = require('chai') +const { expect } = chai +chai.should() -const RealTimeClient = require("./helpers/RealTimeClient"); -const MockWebServer = require("./helpers/MockWebServer"); -const FixturesManager = require("./helpers/FixturesManager"); +const RealTimeClient = require('./helpers/RealTimeClient') +const MockWebServer = require('./helpers/MockWebServer') +const FixturesManager = require('./helpers/FixturesManager') -const async = require("async"); +const async = require('async') -const settings = require("settings-sharelatex"); -const redis = require("redis-sharelatex"); -const rclient = redis.createClient(settings.redis.pubsub); +const settings = require('settings-sharelatex') +const redis = require('redis-sharelatex') +const rclient = redis.createClient(settings.redis.pubsub) -describe("receiveUpdate", function() { - beforeEach(function(done) { - this.lines = ["test", "doc", "lines"]; - this.version = 42; - this.ops = ["mock", "doc", "ops"]; - - return async.series([ - cb => { - return FixturesManager.setUpProject({ - privilegeLevel: "owner", - project: { name: "Test Project" } - }, (error, {user_id, project_id}) => { this.user_id = user_id; this.project_id = project_id; return cb(); }); - }, - - cb => { - return FixturesManager.setUpDoc(this.project_id, {lines: this.lines, version: this.version, ops: this.ops}, (e, {doc_id}) => { - this.doc_id = doc_id; - return cb(e); - }); - }, - - cb => { - this.clientA = RealTimeClient.connect(); - return this.clientA.on("connectionAccepted", cb); - }, - - cb => { - this.clientB = RealTimeClient.connect(); - return this.clientB.on("connectionAccepted", cb); - }, - - cb => { - return this.clientA.emit("joinProject", { - project_id: this.project_id - }, cb); - }, - - cb => { - return this.clientA.emit("joinDoc", this.doc_id, cb); - }, - - cb => { - return this.clientB.emit("joinProject", { - project_id: this.project_id - }, cb); - }, - - cb => { - return this.clientB.emit("joinDoc", this.doc_id, cb); - }, +describe('receiveUpdate', function () { + beforeEach(function (done) { + this.lines = ['test', 'doc', 'lines'] + this.version = 42 + this.ops = ['mock', 'doc', 'ops'] - cb => { - return FixturesManager.setUpProject({ - privilegeLevel: "owner", - project: {name: "Test Project"} - }, (error, {user_id: user_id_second, project_id: project_id_second}) => { this.user_id_second = user_id_second; this.project_id_second = project_id_second; return cb(); }); - }, + return async.series( + [ + (cb) => { + return FixturesManager.setUpProject( + { + privilegeLevel: 'owner', + project: { name: 'Test Project' } + }, + (error, { user_id, project_id }) => { + this.user_id = user_id + this.project_id = project_id + return cb() + } + ) + }, - cb => { - return FixturesManager.setUpDoc(this.project_id_second, {lines: this.lines, version: this.version, ops: this.ops}, (e, {doc_id: doc_id_second}) => { - this.doc_id_second = doc_id_second; - return cb(e); - }); - }, + (cb) => { + return FixturesManager.setUpDoc( + this.project_id, + { lines: this.lines, version: this.version, ops: this.ops }, + (e, { doc_id }) => { + this.doc_id = doc_id + return cb(e) + } + ) + }, - cb => { - this.clientC = RealTimeClient.connect(); - return this.clientC.on("connectionAccepted", cb); - }, + (cb) => { + this.clientA = RealTimeClient.connect() + return this.clientA.on('connectionAccepted', cb) + }, - cb => { - return this.clientC.emit("joinProject", { - project_id: this.project_id_second - }, cb); - }, - cb => { - return this.clientC.emit("joinDoc", this.doc_id_second, cb); - }, + (cb) => { + this.clientB = RealTimeClient.connect() + return this.clientB.on('connectionAccepted', cb) + }, - cb => { - this.clientAUpdates = []; - this.clientA.on("otUpdateApplied", update => this.clientAUpdates.push(update)); - this.clientBUpdates = []; - this.clientB.on("otUpdateApplied", update => this.clientBUpdates.push(update)); - this.clientCUpdates = []; - this.clientC.on("otUpdateApplied", update => this.clientCUpdates.push(update)); + (cb) => { + return this.clientA.emit( + 'joinProject', + { + project_id: this.project_id + }, + cb + ) + }, - this.clientAErrors = []; - this.clientA.on("otUpdateError", error => this.clientAErrors.push(error)); - this.clientBErrors = []; - this.clientB.on("otUpdateError", error => this.clientBErrors.push(error)); - this.clientCErrors = []; - this.clientC.on("otUpdateError", error => this.clientCErrors.push(error)); - return cb(); - } - ], done); - }); + (cb) => { + return this.clientA.emit('joinDoc', this.doc_id, cb) + }, - afterEach(function() { - if (this.clientA != null) { - this.clientA.disconnect(); - } - if (this.clientB != null) { - this.clientB.disconnect(); - } - return (this.clientC != null ? this.clientC.disconnect() : undefined); - }); + (cb) => { + return this.clientB.emit( + 'joinProject', + { + project_id: this.project_id + }, + cb + ) + }, - describe("with an update from clientA", function() { - beforeEach(function(done) { - this.update = { - doc_id: this.doc_id, - op: { - meta: { - source: this.clientA.publicId - }, - v: this.version, - doc: this.doc_id, - op: [{i: "foo", p: 50}] - } - }; - rclient.publish("applied-ops", JSON.stringify(this.update)); - return setTimeout(done, 200); - }); // Give clients time to get message - - it("should send the full op to clientB", function() { - return this.clientBUpdates.should.deep.equal([this.update.op]); - }); - - it("should send an ack to clientA", function() { - return this.clientAUpdates.should.deep.equal([{ - v: this.version, doc: this.doc_id - }]); - }); + (cb) => { + return this.clientB.emit('joinDoc', this.doc_id, cb) + }, - return it("should send nothing to clientC", function() { - return this.clientCUpdates.should.deep.equal([]); - }); -}); + (cb) => { + return FixturesManager.setUpProject( + { + privilegeLevel: 'owner', + project: { name: 'Test Project' } + }, + ( + error, + { user_id: user_id_second, project_id: project_id_second } + ) => { + this.user_id_second = user_id_second + this.project_id_second = project_id_second + return cb() + } + ) + }, - describe("with an update from clientC", function() { - beforeEach(function(done) { - this.update = { - doc_id: this.doc_id_second, - op: { - meta: { - source: this.clientC.publicId - }, - v: this.version, - doc: this.doc_id_second, - op: [{i: "update from clientC", p: 50}] - } - }; - rclient.publish("applied-ops", JSON.stringify(this.update)); - return setTimeout(done, 200); - }); // Give clients time to get message + (cb) => { + return FixturesManager.setUpDoc( + this.project_id_second, + { lines: this.lines, version: this.version, ops: this.ops }, + (e, { doc_id: doc_id_second }) => { + this.doc_id_second = doc_id_second + return cb(e) + } + ) + }, - it("should send nothing to clientA", function() { - return this.clientAUpdates.should.deep.equal([]); - }); + (cb) => { + this.clientC = RealTimeClient.connect() + return this.clientC.on('connectionAccepted', cb) + }, - it("should send nothing to clientB", function() { - return this.clientBUpdates.should.deep.equal([]); - }); + (cb) => { + return this.clientC.emit( + 'joinProject', + { + project_id: this.project_id_second + }, + cb + ) + }, + (cb) => { + return this.clientC.emit('joinDoc', this.doc_id_second, cb) + }, - return it("should send an ack to clientC", function() { - return this.clientCUpdates.should.deep.equal([{ - v: this.version, doc: this.doc_id_second - }]); - }); -}); + (cb) => { + this.clientAUpdates = [] + this.clientA.on('otUpdateApplied', (update) => + this.clientAUpdates.push(update) + ) + this.clientBUpdates = [] + this.clientB.on('otUpdateApplied', (update) => + this.clientBUpdates.push(update) + ) + this.clientCUpdates = [] + this.clientC.on('otUpdateApplied', (update) => + this.clientCUpdates.push(update) + ) - describe("with an update from a remote client for project 1", function() { - beforeEach(function(done) { - this.update = { - doc_id: this.doc_id, - op: { - meta: { - source: 'this-is-a-remote-client-id' - }, - v: this.version, - doc: this.doc_id, - op: [{i: "foo", p: 50}] - } - }; - rclient.publish("applied-ops", JSON.stringify(this.update)); - return setTimeout(done, 200); - }); // Give clients time to get message + this.clientAErrors = [] + this.clientA.on('otUpdateError', (error) => + this.clientAErrors.push(error) + ) + this.clientBErrors = [] + this.clientB.on('otUpdateError', (error) => + this.clientBErrors.push(error) + ) + this.clientCErrors = [] + this.clientC.on('otUpdateError', (error) => + this.clientCErrors.push(error) + ) + return cb() + } + ], + done + ) + }) - it("should send the full op to clientA", function() { - return this.clientAUpdates.should.deep.equal([this.update.op]); - }); - - it("should send the full op to clientB", function() { - return this.clientBUpdates.should.deep.equal([this.update.op]); - }); + afterEach(function () { + if (this.clientA != null) { + this.clientA.disconnect() + } + if (this.clientB != null) { + this.clientB.disconnect() + } + return this.clientC != null ? this.clientC.disconnect() : undefined + }) - return it("should send nothing to clientC", function() { - return this.clientCUpdates.should.deep.equal([]); - }); -}); + describe('with an update from clientA', function () { + beforeEach(function (done) { + this.update = { + doc_id: this.doc_id, + op: { + meta: { + source: this.clientA.publicId + }, + v: this.version, + doc: this.doc_id, + op: [{ i: 'foo', p: 50 }] + } + } + rclient.publish('applied-ops', JSON.stringify(this.update)) + return setTimeout(done, 200) + }) // Give clients time to get message - describe("with an error for the first project", function() { - beforeEach(function(done) { - rclient.publish("applied-ops", JSON.stringify({doc_id: this.doc_id, error: (this.error = "something went wrong")})); - return setTimeout(done, 200); - }); // Give clients time to get message + it('should send the full op to clientB', function () { + return this.clientBUpdates.should.deep.equal([this.update.op]) + }) - it("should send the error to the clients in the first project", function() { - this.clientAErrors.should.deep.equal([this.error]); - return this.clientBErrors.should.deep.equal([this.error]); - }); + it('should send an ack to clientA', function () { + return this.clientAUpdates.should.deep.equal([ + { + v: this.version, + doc: this.doc_id + } + ]) + }) - it("should not send any errors to the client in the second project", function() { - return this.clientCErrors.should.deep.equal([]); - }); + return it('should send nothing to clientC', function () { + return this.clientCUpdates.should.deep.equal([]) + }) + }) - it("should disconnect the clients of the first project", function() { - this.clientA.socket.connected.should.equal(false); - return this.clientB.socket.connected.should.equal(false); - }); + describe('with an update from clientC', function () { + beforeEach(function (done) { + this.update = { + doc_id: this.doc_id_second, + op: { + meta: { + source: this.clientC.publicId + }, + v: this.version, + doc: this.doc_id_second, + op: [{ i: 'update from clientC', p: 50 }] + } + } + rclient.publish('applied-ops', JSON.stringify(this.update)) + return setTimeout(done, 200) + }) // Give clients time to get message - return it("should not disconnect the client in the second project", function() { - return this.clientC.socket.connected.should.equal(true); - }); - }); + it('should send nothing to clientA', function () { + return this.clientAUpdates.should.deep.equal([]) + }) - return describe("with an error for the second project", function() { - beforeEach(function(done) { - rclient.publish("applied-ops", JSON.stringify({doc_id: this.doc_id_second, error: (this.error = "something went wrong")})); - return setTimeout(done, 200); - }); // Give clients time to get message + it('should send nothing to clientB', function () { + return this.clientBUpdates.should.deep.equal([]) + }) - it("should not send any errors to the clients in the first project", function() { - this.clientAErrors.should.deep.equal([]); - return this.clientBErrors.should.deep.equal([]); - }); + return it('should send an ack to clientC', function () { + return this.clientCUpdates.should.deep.equal([ + { + v: this.version, + doc: this.doc_id_second + } + ]) + }) + }) - it("should send the error to the client in the second project", function() { - return this.clientCErrors.should.deep.equal([this.error]); - }); + describe('with an update from a remote client for project 1', function () { + beforeEach(function (done) { + this.update = { + doc_id: this.doc_id, + op: { + meta: { + source: 'this-is-a-remote-client-id' + }, + v: this.version, + doc: this.doc_id, + op: [{ i: 'foo', p: 50 }] + } + } + rclient.publish('applied-ops', JSON.stringify(this.update)) + return setTimeout(done, 200) + }) // Give clients time to get message - it("should not disconnect the clients of the first project", function() { - this.clientA.socket.connected.should.equal(true); - return this.clientB.socket.connected.should.equal(true); - }); + it('should send the full op to clientA', function () { + return this.clientAUpdates.should.deep.equal([this.update.op]) + }) - return it("should disconnect the client in the second project", function() { - return this.clientC.socket.connected.should.equal(false); - }); - }); -}); + it('should send the full op to clientB', function () { + return this.clientBUpdates.should.deep.equal([this.update.op]) + }) + + return it('should send nothing to clientC', function () { + return this.clientCUpdates.should.deep.equal([]) + }) + }) + + describe('with an error for the first project', function () { + beforeEach(function (done) { + rclient.publish( + 'applied-ops', + JSON.stringify({ + doc_id: this.doc_id, + error: (this.error = 'something went wrong') + }) + ) + return setTimeout(done, 200) + }) // Give clients time to get message + + it('should send the error to the clients in the first project', function () { + this.clientAErrors.should.deep.equal([this.error]) + return this.clientBErrors.should.deep.equal([this.error]) + }) + + it('should not send any errors to the client in the second project', function () { + return this.clientCErrors.should.deep.equal([]) + }) + + it('should disconnect the clients of the first project', function () { + this.clientA.socket.connected.should.equal(false) + return this.clientB.socket.connected.should.equal(false) + }) + + return it('should not disconnect the client in the second project', function () { + return this.clientC.socket.connected.should.equal(true) + }) + }) + + return describe('with an error for the second project', function () { + beforeEach(function (done) { + rclient.publish( + 'applied-ops', + JSON.stringify({ + doc_id: this.doc_id_second, + error: (this.error = 'something went wrong') + }) + ) + return setTimeout(done, 200) + }) // Give clients time to get message + + it('should not send any errors to the clients in the first project', function () { + this.clientAErrors.should.deep.equal([]) + return this.clientBErrors.should.deep.equal([]) + }) + + it('should send the error to the client in the second project', function () { + return this.clientCErrors.should.deep.equal([this.error]) + }) + + it('should not disconnect the clients of the first project', function () { + this.clientA.socket.connected.should.equal(true) + return this.clientB.socket.connected.should.equal(true) + }) + + return it('should disconnect the client in the second project', function () { + return this.clientC.socket.connected.should.equal(false) + }) + }) +}) diff --git a/services/real-time/test/acceptance/js/RouterTests.js b/services/real-time/test/acceptance/js/RouterTests.js index 844a4061cf..729947281c 100644 --- a/services/real-time/test/acceptance/js/RouterTests.js +++ b/services/real-time/test/acceptance/js/RouterTests.js @@ -8,99 +8,114 @@ * DS102: Remove unnecessary code created because of implicit returns * Full docs: https://github.com/decaffeinate/decaffeinate/blob/master/docs/suggestions.md */ -const async = require("async"); -const {expect} = require("chai"); +const async = require('async') +const { expect } = require('chai') -const RealTimeClient = require("./helpers/RealTimeClient"); -const FixturesManager = require("./helpers/FixturesManager"); +const RealTimeClient = require('./helpers/RealTimeClient') +const FixturesManager = require('./helpers/FixturesManager') +describe('Router', function () { + return describe('joinProject', function () { + describe('when there is no callback provided', function () { + after(function () { + return process.removeListener('unhandledRejection', this.onUnhandled) + }) -describe("Router", function() { return describe("joinProject", function() { - describe("when there is no callback provided", function() { - after(function() { - return process.removeListener('unhandledRejection', this.onUnhandled); - }); - - before(function(done) { - this.onUnhandled = error => done(error); - process.on('unhandledRejection', this.onUnhandled); - return async.series([ - cb => { - return FixturesManager.setUpProject({ - privilegeLevel: "owner", - project: { - name: "Test Project" - } - }, (e, {project_id, user_id}) => { - this.project_id = project_id; - this.user_id = user_id; - return cb(e); - }); + before(function (done) { + this.onUnhandled = (error) => done(error) + process.on('unhandledRejection', this.onUnhandled) + return async.series( + [ + (cb) => { + return FixturesManager.setUpProject( + { + privilegeLevel: 'owner', + project: { + name: 'Test Project' + } }, - - cb => { - this.client = RealTimeClient.connect(); - return this.client.on("connectionAccepted", cb); - }, - - cb => { - this.client = RealTimeClient.connect(); - return this.client.on("connectionAccepted", cb); - }, - - cb => { - this.client.emit("joinProject", {project_id: this.project_id}); - return setTimeout(cb, 100); + (e, { project_id, user_id }) => { + this.project_id = project_id + this.user_id = user_id + return cb(e) } - ], done); - }); + ) + }, - return it("should keep on going", function() { return expect('still running').to.exist; }); - }); + (cb) => { + this.client = RealTimeClient.connect() + return this.client.on('connectionAccepted', cb) + }, - return describe("when there are too many arguments", function() { - after(function() { - return process.removeListener('unhandledRejection', this.onUnhandled); - }); + (cb) => { + this.client = RealTimeClient.connect() + return this.client.on('connectionAccepted', cb) + }, - before(function(done) { - this.onUnhandled = error => done(error); - process.on('unhandledRejection', this.onUnhandled); - return async.series([ - cb => { - return FixturesManager.setUpProject({ - privilegeLevel: "owner", - project: { - name: "Test Project" - } - }, (e, {project_id, user_id}) => { - this.project_id = project_id; - this.user_id = user_id; - return cb(e); - }); + (cb) => { + this.client.emit('joinProject', { project_id: this.project_id }) + return setTimeout(cb, 100) + } + ], + done + ) + }) + + return it('should keep on going', function () { + return expect('still running').to.exist + }) + }) + + return describe('when there are too many arguments', function () { + after(function () { + return process.removeListener('unhandledRejection', this.onUnhandled) + }) + + before(function (done) { + this.onUnhandled = (error) => done(error) + process.on('unhandledRejection', this.onUnhandled) + return async.series( + [ + (cb) => { + return FixturesManager.setUpProject( + { + privilegeLevel: 'owner', + project: { + name: 'Test Project' + } }, - - cb => { - this.client = RealTimeClient.connect(); - return this.client.on("connectionAccepted", cb); - }, - - cb => { - this.client = RealTimeClient.connect(); - return this.client.on("connectionAccepted", cb); - }, - - cb => { - return this.client.emit("joinProject", 1, 2, 3, 4, 5, error => { - this.error = error; - return cb(); - }); + (e, { project_id, user_id }) => { + this.project_id = project_id + this.user_id = user_id + return cb(e) } - ], done); - }); + ) + }, - return it("should return an error message", function() { - return expect(this.error.message).to.equal('unexpected arguments'); - }); - }); -}); }); + (cb) => { + this.client = RealTimeClient.connect() + return this.client.on('connectionAccepted', cb) + }, + + (cb) => { + this.client = RealTimeClient.connect() + return this.client.on('connectionAccepted', cb) + }, + + (cb) => { + return this.client.emit('joinProject', 1, 2, 3, 4, 5, (error) => { + this.error = error + return cb() + }) + } + ], + done + ) + }) + + return it('should return an error message', function () { + return expect(this.error.message).to.equal('unexpected arguments') + }) + }) + }) +}) diff --git a/services/real-time/test/acceptance/js/SessionSocketsTests.js b/services/real-time/test/acceptance/js/SessionSocketsTests.js index 93f00cd516..45f62195e5 100644 --- a/services/real-time/test/acceptance/js/SessionSocketsTests.js +++ b/services/real-time/test/acceptance/js/SessionSocketsTests.js @@ -8,88 +8,96 @@ * DS102: Remove unnecessary code created because of implicit returns * Full docs: https://github.com/decaffeinate/decaffeinate/blob/master/docs/suggestions.md */ -const RealTimeClient = require("./helpers/RealTimeClient"); -const Settings = require("settings-sharelatex"); -const {expect} = require('chai'); +const RealTimeClient = require('./helpers/RealTimeClient') +const Settings = require('settings-sharelatex') +const { expect } = require('chai') -describe('SessionSockets', function() { - before(function() { - return this.checkSocket = function(fn) { - const client = RealTimeClient.connect(); - client.on('connectionAccepted', fn); - client.on('connectionRejected', fn); - return null; - }; - }); +describe('SessionSockets', function () { + before(function () { + return (this.checkSocket = function (fn) { + const client = RealTimeClient.connect() + client.on('connectionAccepted', fn) + client.on('connectionRejected', fn) + return null + }) + }) - describe('without cookies', function() { - before(function() { return RealTimeClient.cookie = null; }); + describe('without cookies', function () { + before(function () { + return (RealTimeClient.cookie = null) + }) - return it('should return a lookup error', function(done) { - return this.checkSocket((error) => { - expect(error).to.exist; - expect(error.message).to.equal('invalid session'); - return done(); - }); - }); - }); + return it('should return a lookup error', function (done) { + return this.checkSocket((error) => { + expect(error).to.exist + expect(error.message).to.equal('invalid session') + return done() + }) + }) + }) - describe('with a different cookie', function() { - before(function() { return RealTimeClient.cookie = "some.key=someValue"; }); + describe('with a different cookie', function () { + before(function () { + return (RealTimeClient.cookie = 'some.key=someValue') + }) - return it('should return a lookup error', function(done) { - return this.checkSocket((error) => { - expect(error).to.exist; - expect(error.message).to.equal('invalid session'); - return done(); - }); - }); - }); + return it('should return a lookup error', function (done) { + return this.checkSocket((error) => { + expect(error).to.exist + expect(error.message).to.equal('invalid session') + return done() + }) + }) + }) - describe('with an invalid cookie', function() { - before(function(done) { - RealTimeClient.setSession({}, (error) => { - if (error) { return done(error); } - RealTimeClient.cookie = `${Settings.cookieName}=${ - RealTimeClient.cookie.slice(17, 49) - }`; - return done(); - }); - return null; - }); + describe('with an invalid cookie', function () { + before(function (done) { + RealTimeClient.setSession({}, (error) => { + if (error) { + return done(error) + } + RealTimeClient.cookie = `${ + Settings.cookieName + }=${RealTimeClient.cookie.slice(17, 49)}` + return done() + }) + return null + }) - return it('should return a lookup error', function(done) { - return this.checkSocket((error) => { - expect(error).to.exist; - expect(error.message).to.equal('invalid session'); - return done(); - }); - }); - }); + return it('should return a lookup error', function (done) { + return this.checkSocket((error) => { + expect(error).to.exist + expect(error.message).to.equal('invalid session') + return done() + }) + }) + }) - describe('with a valid cookie and no matching session', function() { - before(function() { return RealTimeClient.cookie = `${Settings.cookieName}=unknownId`; }); + describe('with a valid cookie and no matching session', function () { + before(function () { + return (RealTimeClient.cookie = `${Settings.cookieName}=unknownId`) + }) - return it('should return a lookup error', function(done) { - return this.checkSocket((error) => { - expect(error).to.exist; - expect(error.message).to.equal('invalid session'); - return done(); - }); - }); - }); + return it('should return a lookup error', function (done) { + return this.checkSocket((error) => { + expect(error).to.exist + expect(error.message).to.equal('invalid session') + return done() + }) + }) + }) - return describe('with a valid cookie and a matching session', function() { - before(function(done) { - RealTimeClient.setSession({}, done); - return null; - }); + return describe('with a valid cookie and a matching session', function () { + before(function (done) { + RealTimeClient.setSession({}, done) + return null + }) - return it('should not return an error', function(done) { - return this.checkSocket((error) => { - expect(error).to.not.exist; - return done(); - }); - }); - }); -}); + return it('should not return an error', function (done) { + return this.checkSocket((error) => { + expect(error).to.not.exist + return done() + }) + }) + }) +}) diff --git a/services/real-time/test/acceptance/js/SessionTests.js b/services/real-time/test/acceptance/js/SessionTests.js index d9614784f2..941a59f4b9 100644 --- a/services/real-time/test/acceptance/js/SessionTests.js +++ b/services/real-time/test/acceptance/js/SessionTests.js @@ -11,47 +11,51 @@ * DS207: Consider shorter variations of null checks * Full docs: https://github.com/decaffeinate/decaffeinate/blob/master/docs/suggestions.md */ -const chai = require("chai"); -const { - expect -} = chai; +const chai = require('chai') +const { expect } = chai -const RealTimeClient = require("./helpers/RealTimeClient"); +const RealTimeClient = require('./helpers/RealTimeClient') -describe("Session", function() { return describe("with an established session", function() { - before(function(done) { - this.user_id = "mock-user-id"; - RealTimeClient.setSession({ - user: { _id: this.user_id } - }, error => { - if (error != null) { throw error; } - this.client = RealTimeClient.connect(); - return done(); - }); - return null; - }); - - it("should not get disconnected", function(done) { - let disconnected = false; - this.client.on("disconnect", () => disconnected = true); - return setTimeout(() => { - expect(disconnected).to.equal(false); - return done(); +describe('Session', function () { + return describe('with an established session', function () { + before(function (done) { + this.user_id = 'mock-user-id' + RealTimeClient.setSession( + { + user: { _id: this.user_id } + }, + (error) => { + if (error != null) { + throw error + } + this.client = RealTimeClient.connect() + return done() } - , 500); - }); - - return it("should appear in the list of connected clients", function(done) { - return RealTimeClient.getConnectedClients((error, clients) => { - let included = false; - for (const client of Array.from(clients)) { - if (client.client_id === this.client.socket.sessionid) { - included = true; - break; - } - } - expect(included).to.equal(true); - return done(); - }); - }); -}); }); + ) + return null + }) + + it('should not get disconnected', function (done) { + let disconnected = false + this.client.on('disconnect', () => (disconnected = true)) + return setTimeout(() => { + expect(disconnected).to.equal(false) + return done() + }, 500) + }) + + return it('should appear in the list of connected clients', function (done) { + return RealTimeClient.getConnectedClients((error, clients) => { + let included = false + for (const client of Array.from(clients)) { + if (client.client_id === this.client.socket.sessionid) { + included = true + break + } + } + expect(included).to.equal(true) + return done() + }) + }) + }) +}) diff --git a/services/real-time/test/acceptance/js/helpers/FixturesManager.js b/services/real-time/test/acceptance/js/helpers/FixturesManager.js index 81d9dc40af..3e72961cbf 100644 --- a/services/real-time/test/acceptance/js/helpers/FixturesManager.js +++ b/services/real-time/test/acceptance/js/helpers/FixturesManager.js @@ -10,64 +10,110 @@ * DS207: Consider shorter variations of null checks * Full docs: https://github.com/decaffeinate/decaffeinate/blob/master/docs/suggestions.md */ -let FixturesManager; -const RealTimeClient = require("./RealTimeClient"); -const MockWebServer = require("./MockWebServer"); -const MockDocUpdaterServer = require("./MockDocUpdaterServer"); +let FixturesManager +const RealTimeClient = require('./RealTimeClient') +const MockWebServer = require('./MockWebServer') +const MockDocUpdaterServer = require('./MockDocUpdaterServer') -module.exports = (FixturesManager = { - setUpProject(options, callback) { - if (options == null) { options = {}; } - if (callback == null) { callback = function(error, data) {}; } - if (!options.user_id) { options.user_id = FixturesManager.getRandomId(); } - if (!options.project_id) { options.project_id = FixturesManager.getRandomId(); } - if (!options.project) { options.project = { name: "Test Project" }; } - const {project_id, user_id, privilegeLevel, project, publicAccess} = options; - - const privileges = {}; - privileges[user_id] = privilegeLevel; - if (publicAccess) { - privileges["anonymous-user"] = publicAccess; - } - - MockWebServer.createMockProject(project_id, privileges, project); - return MockWebServer.run(error => { - if (error != null) { throw error; } - return RealTimeClient.setSession({ - user: { - _id: user_id, - first_name: "Joe", - last_name: "Bloggs" - } - }, error => { - if (error != null) { throw error; } - return callback(null, {project_id, user_id, privilegeLevel, project}); - }); - }); - }, - - setUpDoc(project_id, options, callback) { - if (options == null) { options = {}; } - if (callback == null) { callback = function(error, data) {}; } - if (!options.doc_id) { options.doc_id = FixturesManager.getRandomId(); } - if (!options.lines) { options.lines = ["doc", "lines"]; } - if (!options.version) { options.version = 42; } - if (!options.ops) { options.ops = ["mock", "ops"]; } - const {doc_id, lines, version, ops, ranges} = options; - - MockDocUpdaterServer.createMockDoc(project_id, doc_id, {lines, version, ops, ranges}); - return MockDocUpdaterServer.run(error => { - if (error != null) { throw error; } - return callback(null, {project_id, doc_id, lines, version, ops}); - }); - }, - - getRandomId() { - return require("crypto") - .createHash("sha1") - .update(Math.random().toString()) - .digest("hex") - .slice(0,24); - } -}); - \ No newline at end of file +module.exports = FixturesManager = { + setUpProject(options, callback) { + if (options == null) { + options = {} + } + if (callback == null) { + callback = function (error, data) {} + } + if (!options.user_id) { + options.user_id = FixturesManager.getRandomId() + } + if (!options.project_id) { + options.project_id = FixturesManager.getRandomId() + } + if (!options.project) { + options.project = { name: 'Test Project' } + } + const { + project_id, + user_id, + privilegeLevel, + project, + publicAccess + } = options + + const privileges = {} + privileges[user_id] = privilegeLevel + if (publicAccess) { + privileges['anonymous-user'] = publicAccess + } + + MockWebServer.createMockProject(project_id, privileges, project) + return MockWebServer.run((error) => { + if (error != null) { + throw error + } + return RealTimeClient.setSession( + { + user: { + _id: user_id, + first_name: 'Joe', + last_name: 'Bloggs' + } + }, + (error) => { + if (error != null) { + throw error + } + return callback(null, { + project_id, + user_id, + privilegeLevel, + project + }) + } + ) + }) + }, + + setUpDoc(project_id, options, callback) { + if (options == null) { + options = {} + } + if (callback == null) { + callback = function (error, data) {} + } + if (!options.doc_id) { + options.doc_id = FixturesManager.getRandomId() + } + if (!options.lines) { + options.lines = ['doc', 'lines'] + } + if (!options.version) { + options.version = 42 + } + if (!options.ops) { + options.ops = ['mock', 'ops'] + } + const { doc_id, lines, version, ops, ranges } = options + + MockDocUpdaterServer.createMockDoc(project_id, doc_id, { + lines, + version, + ops, + ranges + }) + return MockDocUpdaterServer.run((error) => { + if (error != null) { + throw error + } + return callback(null, { project_id, doc_id, lines, version, ops }) + }) + }, + + getRandomId() { + return require('crypto') + .createHash('sha1') + .update(Math.random().toString()) + .digest('hex') + .slice(0, 24) + } +} diff --git a/services/real-time/test/acceptance/js/helpers/MockDocUpdaterServer.js b/services/real-time/test/acceptance/js/helpers/MockDocUpdaterServer.js index 2ce05c4279..f9dcc57bf7 100644 --- a/services/real-time/test/acceptance/js/helpers/MockDocUpdaterServer.js +++ b/services/real-time/test/acceptance/js/helpers/MockDocUpdaterServer.js @@ -11,62 +11,80 @@ * DS207: Consider shorter variations of null checks * Full docs: https://github.com/decaffeinate/decaffeinate/blob/master/docs/suggestions.md */ -let MockDocUpdaterServer; -const sinon = require("sinon"); -const express = require("express"); +let MockDocUpdaterServer +const sinon = require('sinon') +const express = require('express') -module.exports = (MockDocUpdaterServer = { - docs: {}, - - createMockDoc(project_id, doc_id, data) { - return MockDocUpdaterServer.docs[`${project_id}:${doc_id}`] = data; - }, - - getDocument(project_id, doc_id, fromVersion, callback) { - if (callback == null) { callback = function(error, data) {}; } - return callback( - null, MockDocUpdaterServer.docs[`${project_id}:${doc_id}`] - ); - }, - - deleteProject: sinon.stub().callsArg(1), - - getDocumentRequest(req, res, next) { - const {project_id, doc_id} = req.params; - let {fromVersion} = req.query; - fromVersion = parseInt(fromVersion, 10); - return MockDocUpdaterServer.getDocument(project_id, doc_id, fromVersion, (error, data) => { - if (error != null) { return next(error); } - return res.json(data); - }); - }, - - deleteProjectRequest(req, res, next) { - const {project_id} = req.params; - return MockDocUpdaterServer.deleteProject(project_id, (error) => { - if (error != null) { return next(error); } - return res.sendStatus(204); - }); - }, - - running: false, - run(callback) { - if (callback == null) { callback = function(error) {}; } - if (MockDocUpdaterServer.running) { - return callback(); - } - const app = express(); - app.get("/project/:project_id/doc/:doc_id", MockDocUpdaterServer.getDocumentRequest); - app.delete("/project/:project_id", MockDocUpdaterServer.deleteProjectRequest); - return app.listen(3003, (error) => { - MockDocUpdaterServer.running = true; - return callback(error); - }).on("error", (error) => { - console.error("error starting MockDocUpdaterServer:", error.message); - return process.exit(1); - }); - } -}); +module.exports = MockDocUpdaterServer = { + docs: {}, - -sinon.spy(MockDocUpdaterServer, "getDocument"); + createMockDoc(project_id, doc_id, data) { + return (MockDocUpdaterServer.docs[`${project_id}:${doc_id}`] = data) + }, + + getDocument(project_id, doc_id, fromVersion, callback) { + if (callback == null) { + callback = function (error, data) {} + } + return callback(null, MockDocUpdaterServer.docs[`${project_id}:${doc_id}`]) + }, + + deleteProject: sinon.stub().callsArg(1), + + getDocumentRequest(req, res, next) { + const { project_id, doc_id } = req.params + let { fromVersion } = req.query + fromVersion = parseInt(fromVersion, 10) + return MockDocUpdaterServer.getDocument( + project_id, + doc_id, + fromVersion, + (error, data) => { + if (error != null) { + return next(error) + } + return res.json(data) + } + ) + }, + + deleteProjectRequest(req, res, next) { + const { project_id } = req.params + return MockDocUpdaterServer.deleteProject(project_id, (error) => { + if (error != null) { + return next(error) + } + return res.sendStatus(204) + }) + }, + + running: false, + run(callback) { + if (callback == null) { + callback = function (error) {} + } + if (MockDocUpdaterServer.running) { + return callback() + } + const app = express() + app.get( + '/project/:project_id/doc/:doc_id', + MockDocUpdaterServer.getDocumentRequest + ) + app.delete( + '/project/:project_id', + MockDocUpdaterServer.deleteProjectRequest + ) + return app + .listen(3003, (error) => { + MockDocUpdaterServer.running = true + return callback(error) + }) + .on('error', (error) => { + console.error('error starting MockDocUpdaterServer:', error.message) + return process.exit(1) + }) + } +} + +sinon.spy(MockDocUpdaterServer, 'getDocument') diff --git a/services/real-time/test/acceptance/js/helpers/MockWebServer.js b/services/real-time/test/acceptance/js/helpers/MockWebServer.js index ea928a42ab..a2cf5af50b 100644 --- a/services/real-time/test/acceptance/js/helpers/MockWebServer.js +++ b/services/real-time/test/acceptance/js/helpers/MockWebServer.js @@ -11,61 +11,72 @@ * DS207: Consider shorter variations of null checks * Full docs: https://github.com/decaffeinate/decaffeinate/blob/master/docs/suggestions.md */ -let MockWebServer; -const sinon = require("sinon"); -const express = require("express"); +let MockWebServer +const sinon = require('sinon') +const express = require('express') -module.exports = (MockWebServer = { - projects: {}, - privileges: {}, - - createMockProject(project_id, privileges, project) { - MockWebServer.privileges[project_id] = privileges; - return MockWebServer.projects[project_id] = project; - }, - - joinProject(project_id, user_id, callback) { - if (callback == null) { callback = function(error, project, privilegeLevel) {}; } - return callback( - null, - MockWebServer.projects[project_id], - MockWebServer.privileges[project_id][user_id] - ); - }, - - joinProjectRequest(req, res, next) { - const {project_id} = req.params; - const {user_id} = req.query; - if (project_id === 'rate-limited') { - return res.status(429).send(); - } else { - return MockWebServer.joinProject(project_id, user_id, (error, project, privilegeLevel) => { - if (error != null) { return next(error); } - return res.json({ - project, - privilegeLevel - }); - }); - } - }, - - running: false, - run(callback) { - if (callback == null) { callback = function(error) {}; } - if (MockWebServer.running) { - return callback(); - } - const app = express(); - app.post("/project/:project_id/join", MockWebServer.joinProjectRequest); - return app.listen(3000, (error) => { - MockWebServer.running = true; - return callback(error); - }).on("error", (error) => { - console.error("error starting MockWebServer:", error.message); - return process.exit(1); - }); - } -}); +module.exports = MockWebServer = { + projects: {}, + privileges: {}, - -sinon.spy(MockWebServer, "joinProject"); + createMockProject(project_id, privileges, project) { + MockWebServer.privileges[project_id] = privileges + return (MockWebServer.projects[project_id] = project) + }, + + joinProject(project_id, user_id, callback) { + if (callback == null) { + callback = function (error, project, privilegeLevel) {} + } + return callback( + null, + MockWebServer.projects[project_id], + MockWebServer.privileges[project_id][user_id] + ) + }, + + joinProjectRequest(req, res, next) { + const { project_id } = req.params + const { user_id } = req.query + if (project_id === 'rate-limited') { + return res.status(429).send() + } else { + return MockWebServer.joinProject( + project_id, + user_id, + (error, project, privilegeLevel) => { + if (error != null) { + return next(error) + } + return res.json({ + project, + privilegeLevel + }) + } + ) + } + }, + + running: false, + run(callback) { + if (callback == null) { + callback = function (error) {} + } + if (MockWebServer.running) { + return callback() + } + const app = express() + app.post('/project/:project_id/join', MockWebServer.joinProjectRequest) + return app + .listen(3000, (error) => { + MockWebServer.running = true + return callback(error) + }) + .on('error', (error) => { + console.error('error starting MockWebServer:', error.message) + return process.exit(1) + }) + } +} + +sinon.spy(MockWebServer, 'joinProject') diff --git a/services/real-time/test/acceptance/js/helpers/RealTimeClient.js b/services/real-time/test/acceptance/js/helpers/RealTimeClient.js index c5ad0d3c5b..ab8e222d93 100644 --- a/services/real-time/test/acceptance/js/helpers/RealTimeClient.js +++ b/services/real-time/test/acceptance/js/helpers/RealTimeClient.js @@ -11,91 +11,121 @@ * DS207: Consider shorter variations of null checks * Full docs: https://github.com/decaffeinate/decaffeinate/blob/master/docs/suggestions.md */ -let Client; -const { - XMLHttpRequest -} = require("../../libs/XMLHttpRequest"); -const io = require("socket.io-client"); -const async = require("async"); +let Client +const { XMLHttpRequest } = require('../../libs/XMLHttpRequest') +const io = require('socket.io-client') +const async = require('async') -const request = require("request"); -const Settings = require("settings-sharelatex"); -const redis = require("redis-sharelatex"); -const rclient = redis.createClient(Settings.redis.websessions); +const request = require('request') +const Settings = require('settings-sharelatex') +const redis = require('redis-sharelatex') +const rclient = redis.createClient(Settings.redis.websessions) -const uid = require('uid-safe').sync; -const signature = require("cookie-signature"); +const uid = require('uid-safe').sync +const signature = require('cookie-signature') -io.util.request = function() { - const xhr = new XMLHttpRequest(); - const _open = xhr.open; - xhr.open = function() { - _open.apply(xhr, arguments); - if (Client.cookie != null) { - return xhr.setRequestHeader("Cookie", Client.cookie); - } - }; - return xhr; -}; +io.util.request = function () { + const xhr = new XMLHttpRequest() + const _open = xhr.open + xhr.open = function () { + _open.apply(xhr, arguments) + if (Client.cookie != null) { + return xhr.setRequestHeader('Cookie', Client.cookie) + } + } + return xhr +} -module.exports = (Client = { - cookie: null, +module.exports = Client = { + cookie: null, - setSession(session, callback) { - if (callback == null) { callback = function(error) {}; } - const sessionId = uid(24); - session.cookie = {}; - return rclient.set("sess:" + sessionId, JSON.stringify(session), (error) => { - if (error != null) { return callback(error); } - const secret = Settings.security.sessionSecret; - const cookieKey = 's:' + signature.sign(sessionId, secret); - Client.cookie = `${Settings.cookieName}=${cookieKey}`; - return callback(); - }); - }, - - unsetSession(callback) { - if (callback == null) { callback = function(error) {}; } - Client.cookie = null; - return callback(); - }, - - connect(cookie) { - const client = io.connect("http://localhost:3026", {'force new connection': true}); - client.on('connectionAccepted', (_, publicId) => client.publicId = publicId); - return client; - }, - - getConnectedClients(callback) { - if (callback == null) { callback = function(error, clients) {}; } - return request.get({ - url: "http://localhost:3026/clients", - json: true - }, (error, response, data) => callback(error, data)); - }, - - getConnectedClient(client_id, callback) { - if (callback == null) { callback = function(error, clients) {}; } - return request.get({ - url: `http://localhost:3026/clients/${client_id}`, - json: true - }, (error, response, data) => callback(error, data)); - }, + setSession(session, callback) { + if (callback == null) { + callback = function (error) {} + } + const sessionId = uid(24) + session.cookie = {} + return rclient.set( + 'sess:' + sessionId, + JSON.stringify(session), + (error) => { + if (error != null) { + return callback(error) + } + const secret = Settings.security.sessionSecret + const cookieKey = 's:' + signature.sign(sessionId, secret) + Client.cookie = `${Settings.cookieName}=${cookieKey}` + return callback() + } + ) + }, + unsetSession(callback) { + if (callback == null) { + callback = function (error) {} + } + Client.cookie = null + return callback() + }, - disconnectClient(client_id, callback) { - request.post({ - url: `http://localhost:3026/client/${client_id}/disconnect`, - auth: { - user: Settings.internal.realTime.user, - pass: Settings.internal.realTime.pass - } - }, (error, response, data) => callback(error, data)); - return null; - }, + connect(cookie) { + const client = io.connect('http://localhost:3026', { + 'force new connection': true + }) + client.on( + 'connectionAccepted', + (_, publicId) => (client.publicId = publicId) + ) + return client + }, - disconnectAllClients(callback) { - return Client.getConnectedClients((error, clients) => async.each(clients, (clientView, cb) => Client.disconnectClient(clientView.client_id, cb) - , callback)); - } -}); + getConnectedClients(callback) { + if (callback == null) { + callback = function (error, clients) {} + } + return request.get( + { + url: 'http://localhost:3026/clients', + json: true + }, + (error, response, data) => callback(error, data) + ) + }, + + getConnectedClient(client_id, callback) { + if (callback == null) { + callback = function (error, clients) {} + } + return request.get( + { + url: `http://localhost:3026/clients/${client_id}`, + json: true + }, + (error, response, data) => callback(error, data) + ) + }, + + disconnectClient(client_id, callback) { + request.post( + { + url: `http://localhost:3026/client/${client_id}/disconnect`, + auth: { + user: Settings.internal.realTime.user, + pass: Settings.internal.realTime.pass + } + }, + (error, response, data) => callback(error, data) + ) + return null + }, + + disconnectAllClients(callback) { + return Client.getConnectedClients((error, clients) => + async.each( + clients, + (clientView, cb) => Client.disconnectClient(clientView.client_id, cb), + callback + ) + ) + } +} diff --git a/services/real-time/test/acceptance/js/helpers/RealtimeServer.js b/services/real-time/test/acceptance/js/helpers/RealtimeServer.js index 480836d1dd..950c4b966d 100644 --- a/services/real-time/test/acceptance/js/helpers/RealtimeServer.js +++ b/services/real-time/test/acceptance/js/helpers/RealtimeServer.js @@ -12,40 +12,53 @@ * DS207: Consider shorter variations of null checks * Full docs: https://github.com/decaffeinate/decaffeinate/blob/master/docs/suggestions.md */ -const app = require('../../../../app'); -const logger = require("logger-sharelatex"); -const Settings = require("settings-sharelatex"); +const app = require('../../../../app') +const logger = require('logger-sharelatex') +const Settings = require('settings-sharelatex') module.exports = { - running: false, - initing: false, - callbacks: [], - ensureRunning(callback) { - if (callback == null) { callback = function(error) {}; } - if (this.running) { - return callback(); - } else if (this.initing) { - return this.callbacks.push(callback); - } else { - this.initing = true; - this.callbacks.push(callback); - return app.listen(__guard__(Settings.internal != null ? Settings.internal.realtime : undefined, x => x.port), "localhost", error => { - if (error != null) { throw error; } - this.running = true; - logger.log("clsi running in dev mode"); + running: false, + initing: false, + callbacks: [], + ensureRunning(callback) { + if (callback == null) { + callback = function (error) {} + } + if (this.running) { + return callback() + } else if (this.initing) { + return this.callbacks.push(callback) + } else { + this.initing = true + this.callbacks.push(callback) + return app.listen( + __guard__( + Settings.internal != null ? Settings.internal.realtime : undefined, + (x) => x.port + ), + 'localhost', + (error) => { + if (error != null) { + throw error + } + this.running = true + logger.log('clsi running in dev mode') - return (() => { - const result = []; - for (callback of Array.from(this.callbacks)) { - result.push(callback()); - } - return result; - })(); - }); - } - } -}; + return (() => { + const result = [] + for (callback of Array.from(this.callbacks)) { + result.push(callback()) + } + return result + })() + } + ) + } + } +} function __guard__(value, transform) { - return (typeof value !== 'undefined' && value !== null) ? transform(value) : undefined; -} \ No newline at end of file + return typeof value !== 'undefined' && value !== null + ? transform(value) + : undefined +} From 42f55c465166fe37e76d0039e36dc2d82bb9e208 Mon Sep 17 00:00:00 2001 From: decaffeinate Date: Tue, 23 Jun 2020 18:30:46 +0100 Subject: [PATCH 383/491] decaffeinate: rename individual coffee files to js files --- services/real-time/{app.coffee => app.js} | 0 .../config/{settings.defaults.coffee => settings.defaults.js} | 0 2 files changed, 0 insertions(+), 0 deletions(-) rename services/real-time/{app.coffee => app.js} (100%) rename services/real-time/config/{settings.defaults.coffee => settings.defaults.js} (100%) diff --git a/services/real-time/app.coffee b/services/real-time/app.js similarity index 100% rename from services/real-time/app.coffee rename to services/real-time/app.js diff --git a/services/real-time/config/settings.defaults.coffee b/services/real-time/config/settings.defaults.js similarity index 100% rename from services/real-time/config/settings.defaults.coffee rename to services/real-time/config/settings.defaults.js From bdfca5f1550f90a94d3737f392e79a7f043b6d51 Mon Sep 17 00:00:00 2001 From: decaffeinate Date: Tue, 23 Jun 2020 18:30:48 +0100 Subject: [PATCH 384/491] decaffeinate: convert individual files to js --- services/real-time/app.js | 314 ++++++++++-------- .../real-time/config/settings.defaults.js | 132 ++++---- 2 files changed, 249 insertions(+), 197 deletions(-) diff --git a/services/real-time/app.js b/services/real-time/app.js index 9f4c9cc44f..49486a72dd 100644 --- a/services/real-time/app.js +++ b/services/real-time/app.js @@ -1,182 +1,218 @@ -Metrics = require("metrics-sharelatex") -Settings = require "settings-sharelatex" -Metrics.initialize(Settings.appName or "real-time") -async = require("async") -_ = require "underscore" +/* + * decaffeinate suggestions: + * DS102: Remove unnecessary code created because of implicit returns + * DS103: Rewrite code to no longer use __guard__ + * DS207: Consider shorter variations of null checks + * Full docs: https://github.com/decaffeinate/decaffeinate/blob/master/docs/suggestions.md + */ +const Metrics = require("metrics-sharelatex"); +const Settings = require("settings-sharelatex"); +Metrics.initialize(Settings.appName || "real-time"); +const async = require("async"); +const _ = require("underscore"); -logger = require "logger-sharelatex" -logger.initialize("real-time") -Metrics.event_loop.monitor(logger) +const logger = require("logger-sharelatex"); +logger.initialize("real-time"); +Metrics.event_loop.monitor(logger); -express = require("express") -session = require("express-session") -redis = require("redis-sharelatex") -if Settings.sentry?.dsn? - logger.initializeErrorReporting(Settings.sentry.dsn) +const express = require("express"); +const session = require("express-session"); +const redis = require("redis-sharelatex"); +if ((Settings.sentry != null ? Settings.sentry.dsn : undefined) != null) { + logger.initializeErrorReporting(Settings.sentry.dsn); +} -sessionRedisClient = redis.createClient(Settings.redis.websessions) +const sessionRedisClient = redis.createClient(Settings.redis.websessions); -RedisStore = require('connect-redis')(session) -SessionSockets = require('./app/js/SessionSockets') -CookieParser = require("cookie-parser") +const RedisStore = require('connect-redis')(session); +const SessionSockets = require('./app/js/SessionSockets'); +const CookieParser = require("cookie-parser"); -DrainManager = require("./app/js/DrainManager") -HealthCheckManager = require("./app/js/HealthCheckManager") +const DrainManager = require("./app/js/DrainManager"); +const HealthCheckManager = require("./app/js/HealthCheckManager"); -# work around frame handler bug in socket.io v0.9.16 -require("./socket.io.patch.js") -# Set up socket.io server -app = express() +// work around frame handler bug in socket.io v0.9.16 +require("./socket.io.patch.js"); +// Set up socket.io server +const app = express(); -server = require('http').createServer(app) -io = require('socket.io').listen(server) +const server = require('http').createServer(app); +const io = require('socket.io').listen(server); -# Bind to sessions -sessionStore = new RedisStore(client: sessionRedisClient) -cookieParser = CookieParser(Settings.security.sessionSecret) +// Bind to sessions +const sessionStore = new RedisStore({client: sessionRedisClient}); +const cookieParser = CookieParser(Settings.security.sessionSecret); -sessionSockets = new SessionSockets(io, sessionStore, cookieParser, Settings.cookieName) +const sessionSockets = new SessionSockets(io, sessionStore, cookieParser, Settings.cookieName); -Metrics.injectMetricsRoute(app) -app.use(Metrics.http.monitor(logger)) +Metrics.injectMetricsRoute(app); +app.use(Metrics.http.monitor(logger)); -io.configure -> - io.enable('browser client minification') - io.enable('browser client etag') +io.configure(function() { + io.enable('browser client minification'); + io.enable('browser client etag'); - # Fix for Safari 5 error of "Error during WebSocket handshake: location mismatch" - # See http://answers.dotcloud.com/question/578/problem-with-websocket-over-ssl-in-safari-with - io.set('match origin protocol', true) + // Fix for Safari 5 error of "Error during WebSocket handshake: location mismatch" + // See http://answers.dotcloud.com/question/578/problem-with-websocket-over-ssl-in-safari-with + io.set('match origin protocol', true); - # gzip uses a Node 0.8.x method of calling the gzip program which - # doesn't work with 0.6.x - #io.enable('browser client gzip') - io.set('transports', ['websocket', 'flashsocket', 'htmlfile', 'xhr-polling', 'jsonp-polling']) - io.set('log level', 1) + // gzip uses a Node 0.8.x method of calling the gzip program which + // doesn't work with 0.6.x + //io.enable('browser client gzip') + io.set('transports', ['websocket', 'flashsocket', 'htmlfile', 'xhr-polling', 'jsonp-polling']); + return io.set('log level', 1); +}); -app.get "/", (req, res, next) -> - res.send "real-time-sharelatex is alive" +app.get("/", (req, res, next) => res.send("real-time-sharelatex is alive")); -app.get "/status", (req, res, next) -> - if Settings.shutDownInProgress - res.send 503 # Service unavailable - else - res.send "real-time-sharelatex is alive" +app.get("/status", function(req, res, next) { + if (Settings.shutDownInProgress) { + return res.send(503); // Service unavailable + } else { + return res.send("real-time-sharelatex is alive"); + } +}); -app.get "/debug/events", (req, res, next) -> - Settings.debugEvents = parseInt(req.query?.count,10) || 20 - logger.log {count: Settings.debugEvents}, "starting debug mode" - res.send "debug mode will log next #{Settings.debugEvents} events" +app.get("/debug/events", function(req, res, next) { + Settings.debugEvents = parseInt(req.query != null ? req.query.count : undefined,10) || 20; + logger.log({count: Settings.debugEvents}, "starting debug mode"); + return res.send(`debug mode will log next ${Settings.debugEvents} events`); +}); -rclient = require("redis-sharelatex").createClient(Settings.redis.realtime) +const rclient = require("redis-sharelatex").createClient(Settings.redis.realtime); -healthCheck = (req, res, next)-> - rclient.healthCheck (error) -> - if error? - logger.err {err: error}, "failed redis health check" - res.sendStatus 500 - else if HealthCheckManager.isFailing() - status = HealthCheckManager.status() - logger.err {pubSubErrors: status}, "failed pubsub health check" - res.sendStatus 500 - else - res.sendStatus 200 +const healthCheck = (req, res, next) => rclient.healthCheck(function(error) { + if (error != null) { + logger.err({err: error}, "failed redis health check"); + return res.sendStatus(500); + } else if (HealthCheckManager.isFailing()) { + const status = HealthCheckManager.status(); + logger.err({pubSubErrors: status}, "failed pubsub health check"); + return res.sendStatus(500); + } else { + return res.sendStatus(200); + } +}); -app.get "/health_check", healthCheck +app.get("/health_check", healthCheck); -app.get "/health_check/redis", healthCheck +app.get("/health_check/redis", healthCheck); -Router = require "./app/js/Router" -Router.configure(app, io, sessionSockets) +const Router = require("./app/js/Router"); +Router.configure(app, io, sessionSockets); -WebsocketLoadBalancer = require "./app/js/WebsocketLoadBalancer" -WebsocketLoadBalancer.listenForEditorEvents(io) +const WebsocketLoadBalancer = require("./app/js/WebsocketLoadBalancer"); +WebsocketLoadBalancer.listenForEditorEvents(io); -DocumentUpdaterController = require "./app/js/DocumentUpdaterController" -DocumentUpdaterController.listenForUpdatesFromDocumentUpdater(io) +const DocumentUpdaterController = require("./app/js/DocumentUpdaterController"); +DocumentUpdaterController.listenForUpdatesFromDocumentUpdater(io); -port = Settings.internal.realTime.port -host = Settings.internal.realTime.host +const { + port +} = Settings.internal.realTime; +const { + host +} = Settings.internal.realTime; -server.listen port, host, (error) -> - throw error if error? - logger.info "realtime starting up, listening on #{host}:#{port}" +server.listen(port, host, function(error) { + if (error != null) { throw error; } + return logger.info(`realtime starting up, listening on ${host}:${port}`); +}); -# Stop huge stack traces in logs from all the socket.io parsing steps. -Error.stackTraceLimit = 10 +// Stop huge stack traces in logs from all the socket.io parsing steps. +Error.stackTraceLimit = 10; -shutdownCleanly = (signal) -> - connectedClients = io.sockets.clients()?.length - if connectedClients == 0 - logger.warn("no clients connected, exiting") - process.exit() - else - logger.warn {connectedClients}, "clients still connected, not shutting down yet" - setTimeout () -> - shutdownCleanly(signal) - , 30 * 1000 +var shutdownCleanly = function(signal) { + const connectedClients = __guard__(io.sockets.clients(), x => x.length); + if (connectedClients === 0) { + logger.warn("no clients connected, exiting"); + return process.exit(); + } else { + logger.warn({connectedClients}, "clients still connected, not shutting down yet"); + return setTimeout(() => shutdownCleanly(signal) + , 30 * 1000); + } +}; -drainAndShutdown = (signal) -> - if Settings.shutDownInProgress - logger.warn signal: signal, "shutdown already in progress, ignoring signal" - return - else - Settings.shutDownInProgress = true - statusCheckInterval = Settings.statusCheckInterval - if statusCheckInterval - logger.warn signal: signal, "received interrupt, delay drain by #{statusCheckInterval}ms" - setTimeout () -> - logger.warn signal: signal, "received interrupt, starting drain over #{shutdownDrainTimeWindow} mins" - DrainManager.startDrainTimeWindow(io, shutdownDrainTimeWindow) - shutdownCleanly(signal) - , statusCheckInterval +const drainAndShutdown = function(signal) { + if (Settings.shutDownInProgress) { + logger.warn({signal}, "shutdown already in progress, ignoring signal"); + return; + } else { + Settings.shutDownInProgress = true; + const { + statusCheckInterval + } = Settings; + if (statusCheckInterval) { + logger.warn({signal}, `received interrupt, delay drain by ${statusCheckInterval}ms`); + } + return setTimeout(function() { + logger.warn({signal}, `received interrupt, starting drain over ${shutdownDrainTimeWindow} mins`); + DrainManager.startDrainTimeWindow(io, shutdownDrainTimeWindow); + return shutdownCleanly(signal); + } + , statusCheckInterval); + } +}; -Settings.shutDownInProgress = false -if Settings.shutdownDrainTimeWindow? - shutdownDrainTimeWindow = parseInt(Settings.shutdownDrainTimeWindow, 10) - logger.log shutdownDrainTimeWindow: shutdownDrainTimeWindow,"shutdownDrainTimeWindow enabled" - for signal in ['SIGINT', 'SIGHUP', 'SIGQUIT', 'SIGUSR1', 'SIGUSR2', 'SIGTERM', 'SIGABRT'] - process.on signal, drainAndShutdown # signal is passed as argument to event handler +Settings.shutDownInProgress = false; +if (Settings.shutdownDrainTimeWindow != null) { + var shutdownDrainTimeWindow = parseInt(Settings.shutdownDrainTimeWindow, 10); + logger.log({shutdownDrainTimeWindow},"shutdownDrainTimeWindow enabled"); + for (let signal of ['SIGINT', 'SIGHUP', 'SIGQUIT', 'SIGUSR1', 'SIGUSR2', 'SIGTERM', 'SIGABRT']) { + process.on(signal, drainAndShutdown); + } // signal is passed as argument to event handler - # global exception handler - if Settings.errors?.catchUncaughtErrors - process.removeAllListeners('uncaughtException') - process.on 'uncaughtException', (error) -> - if ['EPIPE', 'ECONNRESET'].includes(error.code) - Metrics.inc('disconnected_write', 1, {status: error.code}) - return logger.warn err: error, 'attempted to write to disconnected client' - logger.error err: error, 'uncaught exception' - if Settings.errors?.shutdownOnUncaughtError - drainAndShutdown('SIGABRT') + // global exception handler + if (Settings.errors != null ? Settings.errors.catchUncaughtErrors : undefined) { + process.removeAllListeners('uncaughtException'); + process.on('uncaughtException', function(error) { + if (['EPIPE', 'ECONNRESET'].includes(error.code)) { + Metrics.inc('disconnected_write', 1, {status: error.code}); + return logger.warn({err: error}, 'attempted to write to disconnected client'); + } + logger.error({err: error}, 'uncaught exception'); + if (Settings.errors != null ? Settings.errors.shutdownOnUncaughtError : undefined) { + return drainAndShutdown('SIGABRT'); + } + }); + } +} -if Settings.continualPubsubTraffic - console.log "continualPubsubTraffic enabled" +if (Settings.continualPubsubTraffic) { + console.log("continualPubsubTraffic enabled"); - pubsubClient = redis.createClient(Settings.redis.pubsub) - clusterClient = redis.createClient(Settings.redis.websessions) + const pubsubClient = redis.createClient(Settings.redis.pubsub); + const clusterClient = redis.createClient(Settings.redis.websessions); - publishJob = (channel, callback)-> - checker = new HealthCheckManager(channel) - logger.debug {channel:channel}, "sending pub to keep connection alive" - json = JSON.stringify({health_check:true, key: checker.id, date: new Date().toString()}) - Metrics.summary "redis.publish.#{channel}", json.length - pubsubClient.publish channel, json, (err)-> - if err? - logger.err {err, channel}, "error publishing pubsub traffic to redis" - blob = JSON.stringify({keep: "alive"}) - Metrics.summary "redis.publish.cluster-continual-traffic", blob.length - clusterClient.publish "cluster-continual-traffic", blob, callback + const publishJob = function(channel, callback){ + const checker = new HealthCheckManager(channel); + logger.debug({channel}, "sending pub to keep connection alive"); + const json = JSON.stringify({health_check:true, key: checker.id, date: new Date().toString()}); + Metrics.summary(`redis.publish.${channel}`, json.length); + return pubsubClient.publish(channel, json, function(err){ + if (err != null) { + logger.err({err, channel}, "error publishing pubsub traffic to redis"); + } + const blob = JSON.stringify({keep: "alive"}); + Metrics.summary("redis.publish.cluster-continual-traffic", blob.length); + return clusterClient.publish("cluster-continual-traffic", blob, callback); + }); + }; - runPubSubTraffic = -> - async.map ["applied-ops", "editor-events"], publishJob, (err)-> - setTimeout(runPubSubTraffic, 1000 * 20) + var runPubSubTraffic = () => async.map(["applied-ops", "editor-events"], publishJob, err => setTimeout(runPubSubTraffic, 1000 * 20)); - runPubSubTraffic() + runPubSubTraffic(); +} + +function __guard__(value, transform) { + return (typeof value !== 'undefined' && value !== null) ? transform(value) : undefined; +} \ No newline at end of file diff --git a/services/real-time/config/settings.defaults.js b/services/real-time/config/settings.defaults.js index ee4bd74316..e462afcdbd 100644 --- a/services/real-time/config/settings.defaults.js +++ b/services/real-time/config/settings.defaults.js @@ -1,79 +1,95 @@ -settings = - redis: +const settings = { + redis: { - pubsub: - host: process.env['PUBSUB_REDIS_HOST'] or process.env['REDIS_HOST'] or "localhost" - port: process.env['PUBSUB_REDIS_PORT'] or process.env['REDIS_PORT'] or "6379" - password: process.env["PUBSUB_REDIS_PASSWORD"] or process.env["REDIS_PASSWORD"] or "" - maxRetriesPerRequest: parseInt(process.env["PUBSUB_REDIS_MAX_RETRIES_PER_REQUEST"] or process.env["REDIS_MAX_RETRIES_PER_REQUEST"] or "20") + pubsub: { + host: process.env['PUBSUB_REDIS_HOST'] || process.env['REDIS_HOST'] || "localhost", + port: process.env['PUBSUB_REDIS_PORT'] || process.env['REDIS_PORT'] || "6379", + password: process.env["PUBSUB_REDIS_PASSWORD"] || process.env["REDIS_PASSWORD"] || "", + maxRetriesPerRequest: parseInt(process.env["PUBSUB_REDIS_MAX_RETRIES_PER_REQUEST"] || process.env["REDIS_MAX_RETRIES_PER_REQUEST"] || "20") + }, - realtime: - host: process.env['REAL_TIME_REDIS_HOST'] or process.env['REDIS_HOST'] or "localhost" - port: process.env['REAL_TIME_REDIS_PORT'] or process.env['REDIS_PORT'] or "6379" - password: process.env["REAL_TIME_REDIS_PASSWORD"] or process.env["REDIS_PASSWORD"] or "" - key_schema: - clientsInProject: ({project_id}) -> "clients_in_project:{#{project_id}}" - connectedUser: ({project_id, client_id})-> "connected_user:{#{project_id}}:#{client_id}" - maxRetriesPerRequest: parseInt(process.env["REAL_TIME_REDIS_MAX_RETRIES_PER_REQUEST"] or process.env["REDIS_MAX_RETRIES_PER_REQUEST"] or "20") + realtime: { + host: process.env['REAL_TIME_REDIS_HOST'] || process.env['REDIS_HOST'] || "localhost", + port: process.env['REAL_TIME_REDIS_PORT'] || process.env['REDIS_PORT'] || "6379", + password: process.env["REAL_TIME_REDIS_PASSWORD"] || process.env["REDIS_PASSWORD"] || "", + key_schema: { + clientsInProject({project_id}) { return `clients_in_project:{${project_id}}`; }, + connectedUser({project_id, client_id}){ return `connected_user:{${project_id}}:${client_id}`; } + }, + maxRetriesPerRequest: parseInt(process.env["REAL_TIME_REDIS_MAX_RETRIES_PER_REQUEST"] || process.env["REDIS_MAX_RETRIES_PER_REQUEST"] || "20") + }, - documentupdater: - host: process.env['DOC_UPDATER_REDIS_HOST'] or process.env['REDIS_HOST'] or "localhost" - port: process.env['DOC_UPDATER_REDIS_PORT'] or process.env['REDIS_PORT'] or "6379" - password: process.env["DOC_UPDATER_REDIS_PASSWORD"] or process.env["REDIS_PASSWORD"] or "" - key_schema: - pendingUpdates: ({doc_id}) -> "PendingUpdates:{#{doc_id}}" - maxRetriesPerRequest: parseInt(process.env["DOC_UPDATER_REDIS_MAX_RETRIES_PER_REQUEST"] or process.env["REDIS_MAX_RETRIES_PER_REQUEST"] or "20") + documentupdater: { + host: process.env['DOC_UPDATER_REDIS_HOST'] || process.env['REDIS_HOST'] || "localhost", + port: process.env['DOC_UPDATER_REDIS_PORT'] || process.env['REDIS_PORT'] || "6379", + password: process.env["DOC_UPDATER_REDIS_PASSWORD"] || process.env["REDIS_PASSWORD"] || "", + key_schema: { + pendingUpdates({doc_id}) { return `PendingUpdates:{${doc_id}}`; } + }, + maxRetriesPerRequest: parseInt(process.env["DOC_UPDATER_REDIS_MAX_RETRIES_PER_REQUEST"] || process.env["REDIS_MAX_RETRIES_PER_REQUEST"] || "20") + }, - websessions: - host: process.env['WEB_REDIS_HOST'] or process.env['REDIS_HOST'] or "localhost" - port: process.env['WEB_REDIS_PORT'] or process.env['REDIS_PORT'] or "6379" - password: process.env["WEB_REDIS_PASSWORD"] or process.env["REDIS_PASSWORD"] or "" - maxRetriesPerRequest: parseInt(process.env["WEB_REDIS_MAX_RETRIES_PER_REQUEST"] or process.env["REDIS_MAX_RETRIES_PER_REQUEST"] or "20") + websessions: { + host: process.env['WEB_REDIS_HOST'] || process.env['REDIS_HOST'] || "localhost", + port: process.env['WEB_REDIS_PORT'] || process.env['REDIS_PORT'] || "6379", + password: process.env["WEB_REDIS_PASSWORD"] || process.env["REDIS_PASSWORD"] || "", + maxRetriesPerRequest: parseInt(process.env["WEB_REDIS_MAX_RETRIES_PER_REQUEST"] || process.env["REDIS_MAX_RETRIES_PER_REQUEST"] || "20") + } + }, - internal: - realTime: - port: 3026 - host: process.env['LISTEN_ADDRESS'] or "localhost" - user: "sharelatex" + internal: { + realTime: { + port: 3026, + host: process.env['LISTEN_ADDRESS'] || "localhost", + user: "sharelatex", pass: "password" + } + }, - apis: - web: - url: "http://#{process.env['WEB_API_HOST'] or process.env['WEB_HOST'] or "localhost"}:#{process.env['WEB_API_PORT'] or process.env['WEB_PORT'] or 3000}" - user: process.env['WEB_API_USER'] or "sharelatex" - pass: process.env['WEB_API_PASSWORD'] or "password" - documentupdater: - url: "http://#{process.env['DOCUMENT_UPDATER_HOST'] or process.env['DOCUPDATER_HOST'] or "localhost"}:3003" + apis: { + web: { + url: `http://${process.env['WEB_API_HOST'] || process.env['WEB_HOST'] || "localhost"}:${process.env['WEB_API_PORT'] || process.env['WEB_PORT'] || 3000}`, + user: process.env['WEB_API_USER'] || "sharelatex", + pass: process.env['WEB_API_PASSWORD'] || "password" + }, + documentupdater: { + url: `http://${process.env['DOCUMENT_UPDATER_HOST'] || process.env['DOCUPDATER_HOST'] || "localhost"}:3003` + } + }, - security: - sessionSecret: process.env['SESSION_SECRET'] or "secret-please-change" + security: { + sessionSecret: process.env['SESSION_SECRET'] || "secret-please-change" + }, - cookieName: process.env['COOKIE_NAME'] or "sharelatex.sid" + cookieName: process.env['COOKIE_NAME'] || "sharelatex.sid", - max_doc_length: 2 * 1024 * 1024 # 2mb + max_doc_length: 2 * 1024 * 1024, // 2mb - # combine - # max_doc_length (2mb see above) * 2 (delete + insert) - # max_ranges_size (3mb see MAX_RANGES_SIZE in document-updater) - # overhead for JSON serialization - maxUpdateSize: parseInt(process.env['MAX_UPDATE_SIZE']) or 7 * 1024 * 1024 + 64 * 1024 + // combine + // max_doc_length (2mb see above) * 2 (delete + insert) + // max_ranges_size (3mb see MAX_RANGES_SIZE in document-updater) + // overhead for JSON serialization + maxUpdateSize: parseInt(process.env['MAX_UPDATE_SIZE']) || ((7 * 1024 * 1024) + (64 * 1024)), - shutdownDrainTimeWindow: process.env['SHUTDOWN_DRAIN_TIME_WINDOW'] or 9 + shutdownDrainTimeWindow: process.env['SHUTDOWN_DRAIN_TIME_WINDOW'] || 9, - continualPubsubTraffic: process.env['CONTINUAL_PUBSUB_TRAFFIC'] or false + continualPubsubTraffic: process.env['CONTINUAL_PUBSUB_TRAFFIC'] || false, - checkEventOrder: process.env['CHECK_EVENT_ORDER'] or false + checkEventOrder: process.env['CHECK_EVENT_ORDER'] || false, - publishOnIndividualChannels: process.env['PUBLISH_ON_INDIVIDUAL_CHANNELS'] or false + publishOnIndividualChannels: process.env['PUBLISH_ON_INDIVIDUAL_CHANNELS'] || false, - statusCheckInterval: parseInt(process.env['STATUS_CHECK_INTERVAL'] or '0') + statusCheckInterval: parseInt(process.env['STATUS_CHECK_INTERVAL'] || '0'), - sentry: + sentry: { dsn: process.env.SENTRY_DSN + }, - errors: - catchUncaughtErrors: true + errors: { + catchUncaughtErrors: true, shutdownOnUncaughtError: true + } +}; -# console.log settings.redis -module.exports = settings +// console.log settings.redis +module.exports = settings; From 92dede867ff2c39d3719d61dda3210b790a614f1 Mon Sep 17 00:00:00 2001 From: decaffeinate Date: Tue, 23 Jun 2020 18:30:51 +0100 Subject: [PATCH 385/491] prettier: convert individual decaffeinated files to Prettier format --- services/real-time/app.js | 355 ++++++++++-------- .../real-time/config/settings.defaults.js | 208 ++++++---- 2 files changed, 322 insertions(+), 241 deletions(-) diff --git a/services/real-time/app.js b/services/real-time/app.js index 49486a72dd..47cae86f45 100644 --- a/services/real-time/app.js +++ b/services/real-time/app.js @@ -5,214 +5,249 @@ * DS207: Consider shorter variations of null checks * Full docs: https://github.com/decaffeinate/decaffeinate/blob/master/docs/suggestions.md */ -const Metrics = require("metrics-sharelatex"); -const Settings = require("settings-sharelatex"); -Metrics.initialize(Settings.appName || "real-time"); -const async = require("async"); -const _ = require("underscore"); +const Metrics = require('metrics-sharelatex') +const Settings = require('settings-sharelatex') +Metrics.initialize(Settings.appName || 'real-time') +const async = require('async') +const _ = require('underscore') -const logger = require("logger-sharelatex"); -logger.initialize("real-time"); -Metrics.event_loop.monitor(logger); +const logger = require('logger-sharelatex') +logger.initialize('real-time') +Metrics.event_loop.monitor(logger) -const express = require("express"); -const session = require("express-session"); -const redis = require("redis-sharelatex"); +const express = require('express') +const session = require('express-session') +const redis = require('redis-sharelatex') if ((Settings.sentry != null ? Settings.sentry.dsn : undefined) != null) { - logger.initializeErrorReporting(Settings.sentry.dsn); + logger.initializeErrorReporting(Settings.sentry.dsn) } -const sessionRedisClient = redis.createClient(Settings.redis.websessions); +const sessionRedisClient = redis.createClient(Settings.redis.websessions) -const RedisStore = require('connect-redis')(session); -const SessionSockets = require('./app/js/SessionSockets'); -const CookieParser = require("cookie-parser"); +const RedisStore = require('connect-redis')(session) +const SessionSockets = require('./app/js/SessionSockets') +const CookieParser = require('cookie-parser') -const DrainManager = require("./app/js/DrainManager"); -const HealthCheckManager = require("./app/js/HealthCheckManager"); +const DrainManager = require('./app/js/DrainManager') +const HealthCheckManager = require('./app/js/HealthCheckManager') // work around frame handler bug in socket.io v0.9.16 -require("./socket.io.patch.js"); +require('./socket.io.patch.js') // Set up socket.io server -const app = express(); +const app = express() -const server = require('http').createServer(app); -const io = require('socket.io').listen(server); +const server = require('http').createServer(app) +const io = require('socket.io').listen(server) // Bind to sessions -const sessionStore = new RedisStore({client: sessionRedisClient}); -const cookieParser = CookieParser(Settings.security.sessionSecret); +const sessionStore = new RedisStore({ client: sessionRedisClient }) +const cookieParser = CookieParser(Settings.security.sessionSecret) -const sessionSockets = new SessionSockets(io, sessionStore, cookieParser, Settings.cookieName); +const sessionSockets = new SessionSockets( + io, + sessionStore, + cookieParser, + Settings.cookieName +) -Metrics.injectMetricsRoute(app); -app.use(Metrics.http.monitor(logger)); +Metrics.injectMetricsRoute(app) +app.use(Metrics.http.monitor(logger)) -io.configure(function() { - io.enable('browser client minification'); - io.enable('browser client etag'); +io.configure(function () { + io.enable('browser client minification') + io.enable('browser client etag') - // Fix for Safari 5 error of "Error during WebSocket handshake: location mismatch" - // See http://answers.dotcloud.com/question/578/problem-with-websocket-over-ssl-in-safari-with - io.set('match origin protocol', true); + // Fix for Safari 5 error of "Error during WebSocket handshake: location mismatch" + // See http://answers.dotcloud.com/question/578/problem-with-websocket-over-ssl-in-safari-with + io.set('match origin protocol', true) - // gzip uses a Node 0.8.x method of calling the gzip program which - // doesn't work with 0.6.x - //io.enable('browser client gzip') - io.set('transports', ['websocket', 'flashsocket', 'htmlfile', 'xhr-polling', 'jsonp-polling']); - return io.set('log level', 1); -}); + // gzip uses a Node 0.8.x method of calling the gzip program which + // doesn't work with 0.6.x + // io.enable('browser client gzip') + io.set('transports', [ + 'websocket', + 'flashsocket', + 'htmlfile', + 'xhr-polling', + 'jsonp-polling' + ]) + return io.set('log level', 1) +}) -app.get("/", (req, res, next) => res.send("real-time-sharelatex is alive")); +app.get('/', (req, res, next) => res.send('real-time-sharelatex is alive')) -app.get("/status", function(req, res, next) { - if (Settings.shutDownInProgress) { - return res.send(503); // Service unavailable - } else { - return res.send("real-time-sharelatex is alive"); - } -}); +app.get('/status', function (req, res, next) { + if (Settings.shutDownInProgress) { + return res.send(503) // Service unavailable + } else { + return res.send('real-time-sharelatex is alive') + } +}) -app.get("/debug/events", function(req, res, next) { - Settings.debugEvents = parseInt(req.query != null ? req.query.count : undefined,10) || 20; - logger.log({count: Settings.debugEvents}, "starting debug mode"); - return res.send(`debug mode will log next ${Settings.debugEvents} events`); -}); +app.get('/debug/events', function (req, res, next) { + Settings.debugEvents = + parseInt(req.query != null ? req.query.count : undefined, 10) || 20 + logger.log({ count: Settings.debugEvents }, 'starting debug mode') + return res.send(`debug mode will log next ${Settings.debugEvents} events`) +}) -const rclient = require("redis-sharelatex").createClient(Settings.redis.realtime); +const rclient = require('redis-sharelatex').createClient( + Settings.redis.realtime +) -const healthCheck = (req, res, next) => rclient.healthCheck(function(error) { +const healthCheck = (req, res, next) => + rclient.healthCheck(function (error) { if (error != null) { - logger.err({err: error}, "failed redis health check"); - return res.sendStatus(500); + logger.err({ err: error }, 'failed redis health check') + return res.sendStatus(500) } else if (HealthCheckManager.isFailing()) { - const status = HealthCheckManager.status(); - logger.err({pubSubErrors: status}, "failed pubsub health check"); - return res.sendStatus(500); + const status = HealthCheckManager.status() + logger.err({ pubSubErrors: status }, 'failed pubsub health check') + return res.sendStatus(500) } else { - return res.sendStatus(200); + return res.sendStatus(200) } -}); + }) -app.get("/health_check", healthCheck); +app.get('/health_check', healthCheck) -app.get("/health_check/redis", healthCheck); +app.get('/health_check/redis', healthCheck) +const Router = require('./app/js/Router') +Router.configure(app, io, sessionSockets) +const WebsocketLoadBalancer = require('./app/js/WebsocketLoadBalancer') +WebsocketLoadBalancer.listenForEditorEvents(io) -const Router = require("./app/js/Router"); -Router.configure(app, io, sessionSockets); +const DocumentUpdaterController = require('./app/js/DocumentUpdaterController') +DocumentUpdaterController.listenForUpdatesFromDocumentUpdater(io) -const WebsocketLoadBalancer = require("./app/js/WebsocketLoadBalancer"); -WebsocketLoadBalancer.listenForEditorEvents(io); +const { port } = Settings.internal.realTime +const { host } = Settings.internal.realTime -const DocumentUpdaterController = require("./app/js/DocumentUpdaterController"); -DocumentUpdaterController.listenForUpdatesFromDocumentUpdater(io); - -const { - port -} = Settings.internal.realTime; -const { - host -} = Settings.internal.realTime; - -server.listen(port, host, function(error) { - if (error != null) { throw error; } - return logger.info(`realtime starting up, listening on ${host}:${port}`); -}); +server.listen(port, host, function (error) { + if (error != null) { + throw error + } + return logger.info(`realtime starting up, listening on ${host}:${port}`) +}) // Stop huge stack traces in logs from all the socket.io parsing steps. -Error.stackTraceLimit = 10; +Error.stackTraceLimit = 10 +var shutdownCleanly = function (signal) { + const connectedClients = __guard__(io.sockets.clients(), (x) => x.length) + if (connectedClients === 0) { + logger.warn('no clients connected, exiting') + return process.exit() + } else { + logger.warn( + { connectedClients }, + 'clients still connected, not shutting down yet' + ) + return setTimeout(() => shutdownCleanly(signal), 30 * 1000) + } +} -var shutdownCleanly = function(signal) { - const connectedClients = __guard__(io.sockets.clients(), x => x.length); - if (connectedClients === 0) { - logger.warn("no clients connected, exiting"); - return process.exit(); - } else { - logger.warn({connectedClients}, "clients still connected, not shutting down yet"); - return setTimeout(() => shutdownCleanly(signal) - , 30 * 1000); - } -}; +const drainAndShutdown = function (signal) { + if (Settings.shutDownInProgress) { + logger.warn({ signal }, 'shutdown already in progress, ignoring signal') + } else { + Settings.shutDownInProgress = true + const { statusCheckInterval } = Settings + if (statusCheckInterval) { + logger.warn( + { signal }, + `received interrupt, delay drain by ${statusCheckInterval}ms` + ) + } + return setTimeout(function () { + logger.warn( + { signal }, + `received interrupt, starting drain over ${shutdownDrainTimeWindow} mins` + ) + DrainManager.startDrainTimeWindow(io, shutdownDrainTimeWindow) + return shutdownCleanly(signal) + }, statusCheckInterval) + } +} -const drainAndShutdown = function(signal) { - if (Settings.shutDownInProgress) { - logger.warn({signal}, "shutdown already in progress, ignoring signal"); - return; - } else { - Settings.shutDownInProgress = true; - const { - statusCheckInterval - } = Settings; - if (statusCheckInterval) { - logger.warn({signal}, `received interrupt, delay drain by ${statusCheckInterval}ms`); - } - return setTimeout(function() { - logger.warn({signal}, `received interrupt, starting drain over ${shutdownDrainTimeWindow} mins`); - DrainManager.startDrainTimeWindow(io, shutdownDrainTimeWindow); - return shutdownCleanly(signal); - } - , statusCheckInterval); - } -}; - - -Settings.shutDownInProgress = false; +Settings.shutDownInProgress = false if (Settings.shutdownDrainTimeWindow != null) { - var shutdownDrainTimeWindow = parseInt(Settings.shutdownDrainTimeWindow, 10); - logger.log({shutdownDrainTimeWindow},"shutdownDrainTimeWindow enabled"); - for (let signal of ['SIGINT', 'SIGHUP', 'SIGQUIT', 'SIGUSR1', 'SIGUSR2', 'SIGTERM', 'SIGABRT']) { - process.on(signal, drainAndShutdown); - } // signal is passed as argument to event handler + var shutdownDrainTimeWindow = parseInt(Settings.shutdownDrainTimeWindow, 10) + logger.log({ shutdownDrainTimeWindow }, 'shutdownDrainTimeWindow enabled') + for (const signal of [ + 'SIGINT', + 'SIGHUP', + 'SIGQUIT', + 'SIGUSR1', + 'SIGUSR2', + 'SIGTERM', + 'SIGABRT' + ]) { + process.on(signal, drainAndShutdown) + } // signal is passed as argument to event handler - // global exception handler - if (Settings.errors != null ? Settings.errors.catchUncaughtErrors : undefined) { - process.removeAllListeners('uncaughtException'); - process.on('uncaughtException', function(error) { - if (['EPIPE', 'ECONNRESET'].includes(error.code)) { - Metrics.inc('disconnected_write', 1, {status: error.code}); - return logger.warn({err: error}, 'attempted to write to disconnected client'); - } - logger.error({err: error}, 'uncaught exception'); - if (Settings.errors != null ? Settings.errors.shutdownOnUncaughtError : undefined) { - return drainAndShutdown('SIGABRT'); - } - }); - } + // global exception handler + if ( + Settings.errors != null ? Settings.errors.catchUncaughtErrors : undefined + ) { + process.removeAllListeners('uncaughtException') + process.on('uncaughtException', function (error) { + if (['EPIPE', 'ECONNRESET'].includes(error.code)) { + Metrics.inc('disconnected_write', 1, { status: error.code }) + return logger.warn( + { err: error }, + 'attempted to write to disconnected client' + ) + } + logger.error({ err: error }, 'uncaught exception') + if ( + Settings.errors != null + ? Settings.errors.shutdownOnUncaughtError + : undefined + ) { + return drainAndShutdown('SIGABRT') + } + }) + } } if (Settings.continualPubsubTraffic) { - console.log("continualPubsubTraffic enabled"); + console.log('continualPubsubTraffic enabled') - const pubsubClient = redis.createClient(Settings.redis.pubsub); - const clusterClient = redis.createClient(Settings.redis.websessions); + const pubsubClient = redis.createClient(Settings.redis.pubsub) + const clusterClient = redis.createClient(Settings.redis.websessions) - const publishJob = function(channel, callback){ - const checker = new HealthCheckManager(channel); - logger.debug({channel}, "sending pub to keep connection alive"); - const json = JSON.stringify({health_check:true, key: checker.id, date: new Date().toString()}); - Metrics.summary(`redis.publish.${channel}`, json.length); - return pubsubClient.publish(channel, json, function(err){ - if (err != null) { - logger.err({err, channel}, "error publishing pubsub traffic to redis"); - } - const blob = JSON.stringify({keep: "alive"}); - Metrics.summary("redis.publish.cluster-continual-traffic", blob.length); - return clusterClient.publish("cluster-continual-traffic", blob, callback); - }); - }; + const publishJob = function (channel, callback) { + const checker = new HealthCheckManager(channel) + logger.debug({ channel }, 'sending pub to keep connection alive') + const json = JSON.stringify({ + health_check: true, + key: checker.id, + date: new Date().toString() + }) + Metrics.summary(`redis.publish.${channel}`, json.length) + return pubsubClient.publish(channel, json, function (err) { + if (err != null) { + logger.err({ err, channel }, 'error publishing pubsub traffic to redis') + } + const blob = JSON.stringify({ keep: 'alive' }) + Metrics.summary('redis.publish.cluster-continual-traffic', blob.length) + return clusterClient.publish('cluster-continual-traffic', blob, callback) + }) + } + var runPubSubTraffic = () => + async.map(['applied-ops', 'editor-events'], publishJob, (err) => + setTimeout(runPubSubTraffic, 1000 * 20) + ) - var runPubSubTraffic = () => async.map(["applied-ops", "editor-events"], publishJob, err => setTimeout(runPubSubTraffic, 1000 * 20)); - - runPubSubTraffic(); + runPubSubTraffic() } - - - function __guard__(value, transform) { - return (typeof value !== 'undefined' && value !== null) ? transform(value) : undefined; -} \ No newline at end of file + return typeof value !== 'undefined' && value !== null + ? transform(value) + : undefined +} diff --git a/services/real-time/config/settings.defaults.js b/services/real-time/config/settings.defaults.js index e462afcdbd..8f3d562f8b 100644 --- a/services/real-time/config/settings.defaults.js +++ b/services/real-time/config/settings.defaults.js @@ -1,95 +1,141 @@ const settings = { - redis: { + redis: { + pubsub: { + host: + process.env.PUBSUB_REDIS_HOST || process.env.REDIS_HOST || 'localhost', + port: process.env.PUBSUB_REDIS_PORT || process.env.REDIS_PORT || '6379', + password: + process.env.PUBSUB_REDIS_PASSWORD || process.env.REDIS_PASSWORD || '', + maxRetriesPerRequest: parseInt( + process.env.PUBSUB_REDIS_MAX_RETRIES_PER_REQUEST || + process.env.REDIS_MAX_RETRIES_PER_REQUEST || + '20' + ) + }, - pubsub: { - host: process.env['PUBSUB_REDIS_HOST'] || process.env['REDIS_HOST'] || "localhost", - port: process.env['PUBSUB_REDIS_PORT'] || process.env['REDIS_PORT'] || "6379", - password: process.env["PUBSUB_REDIS_PASSWORD"] || process.env["REDIS_PASSWORD"] || "", - maxRetriesPerRequest: parseInt(process.env["PUBSUB_REDIS_MAX_RETRIES_PER_REQUEST"] || process.env["REDIS_MAX_RETRIES_PER_REQUEST"] || "20") - }, + realtime: { + host: + process.env.REAL_TIME_REDIS_HOST || + process.env.REDIS_HOST || + 'localhost', + port: + process.env.REAL_TIME_REDIS_PORT || process.env.REDIS_PORT || '6379', + password: + process.env.REAL_TIME_REDIS_PASSWORD || + process.env.REDIS_PASSWORD || + '', + key_schema: { + clientsInProject({ project_id }) { + return `clients_in_project:{${project_id}}` + }, + connectedUser({ project_id, client_id }) { + return `connected_user:{${project_id}}:${client_id}` + } + }, + maxRetriesPerRequest: parseInt( + process.env.REAL_TIME_REDIS_MAX_RETRIES_PER_REQUEST || + process.env.REDIS_MAX_RETRIES_PER_REQUEST || + '20' + ) + }, - realtime: { - host: process.env['REAL_TIME_REDIS_HOST'] || process.env['REDIS_HOST'] || "localhost", - port: process.env['REAL_TIME_REDIS_PORT'] || process.env['REDIS_PORT'] || "6379", - password: process.env["REAL_TIME_REDIS_PASSWORD"] || process.env["REDIS_PASSWORD"] || "", - key_schema: { - clientsInProject({project_id}) { return `clients_in_project:{${project_id}}`; }, - connectedUser({project_id, client_id}){ return `connected_user:{${project_id}}:${client_id}`; } - }, - maxRetriesPerRequest: parseInt(process.env["REAL_TIME_REDIS_MAX_RETRIES_PER_REQUEST"] || process.env["REDIS_MAX_RETRIES_PER_REQUEST"] || "20") - }, + documentupdater: { + host: + process.env.DOC_UPDATER_REDIS_HOST || + process.env.REDIS_HOST || + 'localhost', + port: + process.env.DOC_UPDATER_REDIS_PORT || process.env.REDIS_PORT || '6379', + password: + process.env.DOC_UPDATER_REDIS_PASSWORD || + process.env.REDIS_PASSWORD || + '', + key_schema: { + pendingUpdates({ doc_id }) { + return `PendingUpdates:{${doc_id}}` + } + }, + maxRetriesPerRequest: parseInt( + process.env.DOC_UPDATER_REDIS_MAX_RETRIES_PER_REQUEST || + process.env.REDIS_MAX_RETRIES_PER_REQUEST || + '20' + ) + }, - documentupdater: { - host: process.env['DOC_UPDATER_REDIS_HOST'] || process.env['REDIS_HOST'] || "localhost", - port: process.env['DOC_UPDATER_REDIS_PORT'] || process.env['REDIS_PORT'] || "6379", - password: process.env["DOC_UPDATER_REDIS_PASSWORD"] || process.env["REDIS_PASSWORD"] || "", - key_schema: { - pendingUpdates({doc_id}) { return `PendingUpdates:{${doc_id}}`; } - }, - maxRetriesPerRequest: parseInt(process.env["DOC_UPDATER_REDIS_MAX_RETRIES_PER_REQUEST"] || process.env["REDIS_MAX_RETRIES_PER_REQUEST"] || "20") - }, + websessions: { + host: process.env.WEB_REDIS_HOST || process.env.REDIS_HOST || 'localhost', + port: process.env.WEB_REDIS_PORT || process.env.REDIS_PORT || '6379', + password: + process.env.WEB_REDIS_PASSWORD || process.env.REDIS_PASSWORD || '', + maxRetriesPerRequest: parseInt( + process.env.WEB_REDIS_MAX_RETRIES_PER_REQUEST || + process.env.REDIS_MAX_RETRIES_PER_REQUEST || + '20' + ) + } + }, - websessions: { - host: process.env['WEB_REDIS_HOST'] || process.env['REDIS_HOST'] || "localhost", - port: process.env['WEB_REDIS_PORT'] || process.env['REDIS_PORT'] || "6379", - password: process.env["WEB_REDIS_PASSWORD"] || process.env["REDIS_PASSWORD"] || "", - maxRetriesPerRequest: parseInt(process.env["WEB_REDIS_MAX_RETRIES_PER_REQUEST"] || process.env["REDIS_MAX_RETRIES_PER_REQUEST"] || "20") - } - }, + internal: { + realTime: { + port: 3026, + host: process.env.LISTEN_ADDRESS || 'localhost', + user: 'sharelatex', + pass: 'password' + } + }, - internal: { - realTime: { - port: 3026, - host: process.env['LISTEN_ADDRESS'] || "localhost", - user: "sharelatex", - pass: "password" - } - }, - - apis: { - web: { - url: `http://${process.env['WEB_API_HOST'] || process.env['WEB_HOST'] || "localhost"}:${process.env['WEB_API_PORT'] || process.env['WEB_PORT'] || 3000}`, - user: process.env['WEB_API_USER'] || "sharelatex", - pass: process.env['WEB_API_PASSWORD'] || "password" - }, - documentupdater: { - url: `http://${process.env['DOCUMENT_UPDATER_HOST'] || process.env['DOCUPDATER_HOST'] || "localhost"}:3003` - } - }, - - security: { - sessionSecret: process.env['SESSION_SECRET'] || "secret-please-change" - }, - - cookieName: process.env['COOKIE_NAME'] || "sharelatex.sid", - - max_doc_length: 2 * 1024 * 1024, // 2mb + apis: { + web: { + url: `http://${ + process.env.WEB_API_HOST || process.env.WEB_HOST || 'localhost' + }:${process.env.WEB_API_PORT || process.env.WEB_PORT || 3000}`, + user: process.env.WEB_API_USER || 'sharelatex', + pass: process.env.WEB_API_PASSWORD || 'password' + }, + documentupdater: { + url: `http://${ + process.env.DOCUMENT_UPDATER_HOST || + process.env.DOCUPDATER_HOST || + 'localhost' + }:3003` + } + }, - // combine - // max_doc_length (2mb see above) * 2 (delete + insert) - // max_ranges_size (3mb see MAX_RANGES_SIZE in document-updater) - // overhead for JSON serialization - maxUpdateSize: parseInt(process.env['MAX_UPDATE_SIZE']) || ((7 * 1024 * 1024) + (64 * 1024)), + security: { + sessionSecret: process.env.SESSION_SECRET || 'secret-please-change' + }, - shutdownDrainTimeWindow: process.env['SHUTDOWN_DRAIN_TIME_WINDOW'] || 9, + cookieName: process.env.COOKIE_NAME || 'sharelatex.sid', - continualPubsubTraffic: process.env['CONTINUAL_PUBSUB_TRAFFIC'] || false, + max_doc_length: 2 * 1024 * 1024, // 2mb - checkEventOrder: process.env['CHECK_EVENT_ORDER'] || false, - - publishOnIndividualChannels: process.env['PUBLISH_ON_INDIVIDUAL_CHANNELS'] || false, + // combine + // max_doc_length (2mb see above) * 2 (delete + insert) + // max_ranges_size (3mb see MAX_RANGES_SIZE in document-updater) + // overhead for JSON serialization + maxUpdateSize: + parseInt(process.env.MAX_UPDATE_SIZE) || 7 * 1024 * 1024 + 64 * 1024, - statusCheckInterval: parseInt(process.env['STATUS_CHECK_INTERVAL'] || '0'), + shutdownDrainTimeWindow: process.env.SHUTDOWN_DRAIN_TIME_WINDOW || 9, - sentry: { - dsn: process.env.SENTRY_DSN - }, + continualPubsubTraffic: process.env.CONTINUAL_PUBSUB_TRAFFIC || false, + + checkEventOrder: process.env.CHECK_EVENT_ORDER || false, + + publishOnIndividualChannels: + process.env.PUBLISH_ON_INDIVIDUAL_CHANNELS || false, + + statusCheckInterval: parseInt(process.env.STATUS_CHECK_INTERVAL || '0'), + + sentry: { + dsn: process.env.SENTRY_DSN + }, + + errors: { + catchUncaughtErrors: true, + shutdownOnUncaughtError: true + } +} - errors: { - catchUncaughtErrors: true, - shutdownOnUncaughtError: true - } -}; - // console.log settings.redis -module.exports = settings; +module.exports = settings From 53058452ee42185235feaecfbd9e3c3b191e78dd Mon Sep 17 00:00:00 2001 From: decaffeinate Date: Wed, 24 Jun 2020 10:28:28 +0100 Subject: [PATCH 386/491] prettier: convert miscellaneous files to Prettier format --- services/real-time/socket.io.patch.js | 52 +- .../test/acceptance/libs/XMLHttpRequest.js | 519 +++++++++--------- 2 files changed, 299 insertions(+), 272 deletions(-) diff --git a/services/real-time/socket.io.patch.js b/services/real-time/socket.io.patch.js index 354d8223c3..c4a0c051ec 100644 --- a/services/real-time/socket.io.patch.js +++ b/services/real-time/socket.io.patch.js @@ -1,51 +1,51 @@ // EventEmitter has been removed from process in node >= 7 // https://github.com/nodejs/node/commit/62b544290a075fe38e233887a06c408ba25a1c71 -if(process.versions.node.split('.')[0] >= 7) { +if (process.versions.node.split('.')[0] >= 7) { process.EventEmitter = require('events') } -var io = require("socket.io"); +var io = require('socket.io') -if (io.version === "0.9.16" || io.version === "0.9.19") { - console.log("patching socket.io hybi-16 transport frame prototype"); - var transports = require("socket.io/lib/transports/websocket/hybi-16.js"); - transports.prototype.frame = patchedFrameHandler; +if (io.version === '0.9.16' || io.version === '0.9.19') { + console.log('patching socket.io hybi-16 transport frame prototype') + var transports = require('socket.io/lib/transports/websocket/hybi-16.js') + transports.prototype.frame = patchedFrameHandler // file hybi-07-12 has the same problem but no browsers are using that protocol now } function patchedFrameHandler(opcode, str) { - var dataBuffer = new Buffer(str), - dataLength = dataBuffer.length, - startOffset = 2, - secondByte = dataLength; + var dataBuffer = new Buffer(str) + var dataLength = dataBuffer.length + var startOffset = 2 + var secondByte = dataLength if (dataLength === 65536) { - console.log("fixing invalid frame length in socket.io"); + console.log('fixing invalid frame length in socket.io') } if (dataLength > 65535) { // original code had > 65536 - startOffset = 10; - secondByte = 127; + startOffset = 10 + secondByte = 127 } else if (dataLength > 125) { - startOffset = 4; - secondByte = 126; + startOffset = 4 + secondByte = 126 } - var outputBuffer = new Buffer(dataLength + startOffset); - outputBuffer[0] = opcode; - outputBuffer[1] = secondByte; - dataBuffer.copy(outputBuffer, startOffset); + var outputBuffer = new Buffer(dataLength + startOffset) + outputBuffer[0] = opcode + outputBuffer[1] = secondByte + dataBuffer.copy(outputBuffer, startOffset) switch (secondByte) { case 126: - outputBuffer[2] = dataLength >>> 8; - outputBuffer[3] = dataLength % 256; - break; + outputBuffer[2] = dataLength >>> 8 + outputBuffer[3] = dataLength % 256 + break case 127: - var l = dataLength; + var l = dataLength for (var i = 1; i <= 8; ++i) { - outputBuffer[startOffset - i] = l & 0xff; - l >>>= 8; + outputBuffer[startOffset - i] = l & 0xff + l >>>= 8 } } - return outputBuffer; + return outputBuffer } const parser = require('socket.io/lib/parser') diff --git a/services/real-time/test/acceptance/libs/XMLHttpRequest.js b/services/real-time/test/acceptance/libs/XMLHttpRequest.js index e79634da5e..21a60ad3bb 100644 --- a/services/real-time/test/acceptance/libs/XMLHttpRequest.js +++ b/services/real-time/test/acceptance/libs/XMLHttpRequest.js @@ -11,100 +11,96 @@ * @license MIT */ -var Url = require("url") - , spawn = require("child_process").spawn - , fs = require('fs'); +var Url = require('url') +var spawn = require('child_process').spawn +var fs = require('fs') -exports.XMLHttpRequest = function() { +exports.XMLHttpRequest = function () { /** * Private variables */ - var self = this; - var http = require('http'); - var https = require('https'); + var self = this + var http = require('http') + var https = require('https') // Holds http.js objects - var client; - var request; - var response; + var client + var request + var response // Request settings - var settings = {}; + var settings = {} // Set some default headers var defaultHeaders = { - "User-Agent": "node-XMLHttpRequest", - "Accept": "*/*", - }; + 'User-Agent': 'node-XMLHttpRequest', + Accept: '*/*' + } - var headers = defaultHeaders; + var headers = defaultHeaders // These headers are not user setable. // The following are allowed but banned in the spec: // * user-agent var forbiddenRequestHeaders = [ - "accept-charset", - "accept-encoding", - "access-control-request-headers", - "access-control-request-method", - "connection", - "content-length", - "content-transfer-encoding", - //"cookie", - "cookie2", - "date", - "expect", - "host", - "keep-alive", - "origin", - "referer", - "te", - "trailer", - "transfer-encoding", - "upgrade", - "via" - ]; + 'accept-charset', + 'accept-encoding', + 'access-control-request-headers', + 'access-control-request-method', + 'connection', + 'content-length', + 'content-transfer-encoding', + // "cookie", + 'cookie2', + 'date', + 'expect', + 'host', + 'keep-alive', + 'origin', + 'referer', + 'te', + 'trailer', + 'transfer-encoding', + 'upgrade', + 'via' + ] // These request methods are not allowed - var forbiddenRequestMethods = [ - "TRACE", - "TRACK", - "CONNECT" - ]; + var forbiddenRequestMethods = ['TRACE', 'TRACK', 'CONNECT'] // Send flag - var sendFlag = false; + var sendFlag = false // Error flag, used when errors occur or abort is called - var errorFlag = false; + var errorFlag = false // Event listeners - var listeners = {}; + var listeners = {} /** * Constants */ - this.UNSENT = 0; - this.OPENED = 1; - this.HEADERS_RECEIVED = 2; - this.LOADING = 3; - this.DONE = 4; + this.UNSENT = 0 + this.OPENED = 1 + this.HEADERS_RECEIVED = 2 + this.LOADING = 3 + this.DONE = 4 /** * Public vars */ // Current state - this.readyState = this.UNSENT; + this.readyState = this.UNSENT // default ready state change handler in case one is not set or is set late - this.onreadystatechange = null; + this.onreadystatechange = null // Result & response - this.responseText = ""; - this.responseXML = ""; - this.status = null; - this.statusText = null; + this.responseText = '' + this.responseXML = '' + this.status = null + this.statusText = null /** * Private methods @@ -116,9 +112,11 @@ exports.XMLHttpRequest = function() { * @param string header Header to validate * @return boolean False if not allowed, otherwise true */ - var isAllowedHttpHeader = function(header) { - return (header && forbiddenRequestHeaders.indexOf(header.toLowerCase()) === -1); - }; + var isAllowedHttpHeader = function (header) { + return ( + header && forbiddenRequestHeaders.indexOf(header.toLowerCase()) === -1 + ) + } /** * Check if the specified method is allowed. @@ -126,9 +124,9 @@ exports.XMLHttpRequest = function() { * @param string method Request method to validate * @return boolean False if not allowed, otherwise true */ - var isAllowedHttpMethod = function(method) { - return (method && forbiddenRequestMethods.indexOf(method) === -1); - }; + var isAllowedHttpMethod = function (method) { + return method && forbiddenRequestMethods.indexOf(method) === -1 + } /** * Public methods @@ -143,26 +141,26 @@ exports.XMLHttpRequest = function() { * @param string user Username for basic authentication (optional) * @param string password Password for basic authentication (optional) */ - this.open = function(method, url, async, user, password) { - this.abort(); - errorFlag = false; + this.open = function (method, url, async, user, password) { + this.abort() + errorFlag = false // Check for valid request method if (!isAllowedHttpMethod(method)) { - throw "SecurityError: Request method not allowed"; - return; + throw 'SecurityError: Request method not allowed' + return } settings = { - "method": method, - "url": url.toString(), - "async": (typeof async !== "boolean" ? true : async), - "user": user || null, - "password": password || null - }; + method: method, + url: url.toString(), + async: typeof async !== 'boolean' ? true : async, + user: user || null, + password: password || null + } - setState(this.OPENED); - }; + setState(this.OPENED) + } /** * Sets a header for the request. @@ -170,19 +168,19 @@ exports.XMLHttpRequest = function() { * @param string header Header name * @param string value Header value */ - this.setRequestHeader = function(header, value) { + this.setRequestHeader = function (header, value) { if (this.readyState != this.OPENED) { - throw "INVALID_STATE_ERR: setRequestHeader can only be called when state is OPEN"; + throw 'INVALID_STATE_ERR: setRequestHeader can only be called when state is OPEN' } if (!isAllowedHttpHeader(header)) { - console.warn('Refused to set unsafe header "' + header + '"'); - return; + console.warn('Refused to set unsafe header "' + header + '"') + return } if (sendFlag) { - throw "INVALID_STATE_ERR: send flag is true"; + throw 'INVALID_STATE_ERR: send flag is true' } - headers[header] = value; - }; + headers[header] = value + } /** * Gets a header from the server response. @@ -190,37 +188,38 @@ exports.XMLHttpRequest = function() { * @param string header Name of header to get. * @return string Text of the header or null if it doesn't exist. */ - this.getResponseHeader = function(header) { - if (typeof header === "string" - && this.readyState > this.OPENED - && response.headers[header.toLowerCase()] - && !errorFlag + this.getResponseHeader = function (header) { + if ( + typeof header === 'string' && + this.readyState > this.OPENED && + response.headers[header.toLowerCase()] && + !errorFlag ) { - return response.headers[header.toLowerCase()]; + return response.headers[header.toLowerCase()] } - return null; - }; + return null + } /** * Gets all the response headers. * * @return string A string with all response headers separated by CR+LF */ - this.getAllResponseHeaders = function() { + this.getAllResponseHeaders = function () { if (this.readyState < this.HEADERS_RECEIVED || errorFlag) { - return ""; + return '' } - var result = ""; + var result = '' for (var i in response.headers) { // Cookie headers are excluded - if (i !== "set-cookie" && i !== "set-cookie2") { - result += i + ": " + response.headers[i] + "\r\n"; + if (i !== 'set-cookie' && i !== 'set-cookie2') { + result += i + ': ' + response.headers[i] + '\r\n' } } - return result.substr(0, result.length - 2); - }; + return result.substr(0, result.length - 2) + } /** * Gets a request header @@ -228,13 +227,13 @@ exports.XMLHttpRequest = function() { * @param string name Name of header to get * @return string Returns the request header or empty string if not set */ - this.getRequestHeader = function(name) { + this.getRequestHeader = function (name) { // @TODO Make this case insensitive - if (typeof name === "string" && headers[name]) { - return headers[name]; + if (typeof name === 'string' && headers[name]) { + return headers[name] } - return ""; + return '' } /** @@ -242,103 +241,104 @@ exports.XMLHttpRequest = function() { * * @param string data Optional data to send as request body. */ - this.send = function(data) { + this.send = function (data) { if (this.readyState != this.OPENED) { - throw "INVALID_STATE_ERR: connection must be opened before send() is called"; + throw 'INVALID_STATE_ERR: connection must be opened before send() is called' } if (sendFlag) { - throw "INVALID_STATE_ERR: send has already been called"; + throw 'INVALID_STATE_ERR: send has already been called' } - var ssl = false, local = false; - var url = Url.parse(settings.url); + var ssl = false + var local = false + var url = Url.parse(settings.url) // Determine the server switch (url.protocol) { case 'https:': - ssl = true; - // SSL & non-SSL both need host, no break here. + ssl = true + // SSL & non-SSL both need host, no break here. case 'http:': - var host = url.hostname; - break; + var host = url.hostname + break case 'file:': - local = true; - break; + local = true + break case undefined: case '': - var host = "localhost"; - break; + var host = 'localhost' + break default: - throw "Protocol not supported."; + throw 'Protocol not supported.' } // Load files off the local filesystem (file://) if (local) { - if (settings.method !== "GET") { - throw "XMLHttpRequest: Only GET method is supported"; + if (settings.method !== 'GET') { + throw 'XMLHttpRequest: Only GET method is supported' } if (settings.async) { - fs.readFile(url.pathname, 'utf8', function(error, data) { + fs.readFile(url.pathname, 'utf8', (error, data) => { if (error) { - self.handleError(error); + self.handleError(error) } else { - self.status = 200; - self.responseText = data; - setState(self.DONE); + self.status = 200 + self.responseText = data + setState(self.DONE) } - }); + }) } else { try { - this.responseText = fs.readFileSync(url.pathname, 'utf8'); - this.status = 200; - setState(self.DONE); - } catch(e) { - this.handleError(e); + this.responseText = fs.readFileSync(url.pathname, 'utf8') + this.status = 200 + setState(self.DONE) + } catch (e) { + this.handleError(e) } } - return; + return } // Default to port 80. If accessing localhost on another port be sure // to use http://localhost:port/path - var port = url.port || (ssl ? 443 : 80); + var port = url.port || (ssl ? 443 : 80) // Add query string if one is used - var uri = url.pathname + (url.search ? url.search : ''); + var uri = url.pathname + (url.search ? url.search : '') // Set the Host header or the server may reject the request - headers["Host"] = host; + headers.Host = host if (!((ssl && port === 443) || port === 80)) { - headers["Host"] += ':' + url.port; + headers.Host += ':' + url.port } // Set Basic Auth if necessary if (settings.user) { - if (typeof settings.password == "undefined") { - settings.password = ""; + if (typeof settings.password === 'undefined') { + settings.password = '' } - var authBuf = new Buffer(settings.user + ":" + settings.password); - headers["Authorization"] = "Basic " + authBuf.toString("base64"); + var authBuf = new Buffer(settings.user + ':' + settings.password) + headers.Authorization = 'Basic ' + authBuf.toString('base64') } // Set content length header - if (settings.method === "GET" || settings.method === "HEAD") { - data = null; + if (settings.method === 'GET' || settings.method === 'HEAD') { + data = null } else if (data) { - headers["Content-Length"] = Buffer.byteLength(data); + headers['Content-Length'] = Buffer.byteLength(data) - if (!headers["Content-Type"]) { - headers["Content-Type"] = "text/plain;charset=UTF-8"; + if (!headers['Content-Type']) { + headers['Content-Type'] = 'text/plain;charset=UTF-8' } - } else if (settings.method === "POST") { + } else if (settings.method === 'POST') { // For a post with no data set Content-Length: 0. // This is required by buggy servers that don't meet the specs. - headers["Content-Length"] = 0; + headers['Content-Length'] = 0 } var options = { @@ -347,202 +347,229 @@ exports.XMLHttpRequest = function() { path: uri, method: settings.method, headers: headers - }; + } // Reset error flag - errorFlag = false; + errorFlag = false // Handle async requests if (settings.async) { // Use the proper protocol - var doRequest = ssl ? https.request : http.request; + var doRequest = ssl ? https.request : http.request // Request is being sent, set send flag - sendFlag = true; + sendFlag = true // As per spec, this is called here for historical reasons. - self.dispatchEvent("readystatechange"); + self.dispatchEvent('readystatechange') // Create the request - request = doRequest(options, function(resp) { - response = resp; - response.setEncoding("utf8"); + request = doRequest(options, (resp) => { + response = resp + response.setEncoding('utf8') - setState(self.HEADERS_RECEIVED); - self.status = response.statusCode; + setState(self.HEADERS_RECEIVED) + self.status = response.statusCode - response.on('data', function(chunk) { + response.on('data', (chunk) => { // Make sure there's some data if (chunk) { - self.responseText += chunk; + self.responseText += chunk } // Don't emit state changes if the connection has been aborted. if (sendFlag) { - setState(self.LOADING); + setState(self.LOADING) } - }); + }) - response.on('end', function() { + response.on('end', () => { if (sendFlag) { // Discard the 'end' event if the connection has been aborted - setState(self.DONE); - sendFlag = false; + setState(self.DONE) + sendFlag = false } - }); + }) - response.on('error', function(error) { - self.handleError(error); - }); - }).on('error', function(error) { - self.handleError(error); - }); + response.on('error', (error) => { + self.handleError(error) + }) + }).on('error', (error) => { + self.handleError(error) + }) // Node 0.4 and later won't accept empty data. Make sure it's needed. if (data) { - request.write(data); + request.write(data) } - request.end(); + request.end() - self.dispatchEvent("loadstart"); - } else { // Synchronous + self.dispatchEvent('loadstart') + } else { + // Synchronous // Create a temporary file for communication with the other Node process - var syncFile = ".node-xmlhttprequest-sync-" + process.pid; - fs.writeFileSync(syncFile, "", "utf8"); + var syncFile = '.node-xmlhttprequest-sync-' + process.pid + fs.writeFileSync(syncFile, '', 'utf8') // The async request the other Node process executes - var execString = "var http = require('http'), https = require('https'), fs = require('fs');" - + "var doRequest = http" + (ssl ? "s" : "") + ".request;" - + "var options = " + JSON.stringify(options) + ";" - + "var responseText = '';" - + "var req = doRequest(options, function(response) {" - + "response.setEncoding('utf8');" - + "response.on('data', function(chunk) {" - + "responseText += chunk;" - + "});" - + "response.on('end', function() {" - + "fs.writeFileSync('" + syncFile + "', 'NODE-XMLHTTPREQUEST-STATUS:' + response.statusCode + ',' + responseText, 'utf8');" - + "});" - + "response.on('error', function(error) {" - + "fs.writeFileSync('" + syncFile + "', 'NODE-XMLHTTPREQUEST-ERROR:' + JSON.stringify(error), 'utf8');" - + "});" - + "}).on('error', function(error) {" - + "fs.writeFileSync('" + syncFile + "', 'NODE-XMLHTTPREQUEST-ERROR:' + JSON.stringify(error), 'utf8');" - + "});" - + (data ? "req.write('" + data.replace(/'/g, "\\'") + "');":"") - + "req.end();"; + var execString = + "var http = require('http'), https = require('https'), fs = require('fs');" + + 'var doRequest = http' + + (ssl ? 's' : '') + + '.request;' + + 'var options = ' + + JSON.stringify(options) + + ';' + + "var responseText = '';" + + 'var req = doRequest(options, function(response) {' + + "response.setEncoding('utf8');" + + "response.on('data', function(chunk) {" + + 'responseText += chunk;' + + '});' + + "response.on('end', function() {" + + "fs.writeFileSync('" + + syncFile + + "', 'NODE-XMLHTTPREQUEST-STATUS:' + response.statusCode + ',' + responseText, 'utf8');" + + '});' + + "response.on('error', function(error) {" + + "fs.writeFileSync('" + + syncFile + + "', 'NODE-XMLHTTPREQUEST-ERROR:' + JSON.stringify(error), 'utf8');" + + '});' + + "}).on('error', function(error) {" + + "fs.writeFileSync('" + + syncFile + + "', 'NODE-XMLHTTPREQUEST-ERROR:' + JSON.stringify(error), 'utf8');" + + '});' + + (data ? "req.write('" + data.replace(/'/g, "\\'") + "');" : '') + + 'req.end();' // Start the other Node Process, executing this string - syncProc = spawn(process.argv[0], ["-e", execString]); - while((self.responseText = fs.readFileSync(syncFile, 'utf8')) == "") { + syncProc = spawn(process.argv[0], ['-e', execString]) + while ((self.responseText = fs.readFileSync(syncFile, 'utf8')) == '') { // Wait while the file is empty } // Kill the child process once the file has data - syncProc.stdin.end(); + syncProc.stdin.end() // Remove the temporary file - fs.unlinkSync(syncFile); + fs.unlinkSync(syncFile) if (self.responseText.match(/^NODE-XMLHTTPREQUEST-ERROR:/)) { // If the file returned an error, handle it - var errorObj = self.responseText.replace(/^NODE-XMLHTTPREQUEST-ERROR:/, ""); - self.handleError(errorObj); + var errorObj = self.responseText.replace( + /^NODE-XMLHTTPREQUEST-ERROR:/, + '' + ) + self.handleError(errorObj) } else { // If the file returned okay, parse its data and move to the DONE state - self.status = self.responseText.replace(/^NODE-XMLHTTPREQUEST-STATUS:([0-9]*),.*/, "$1"); - self.responseText = self.responseText.replace(/^NODE-XMLHTTPREQUEST-STATUS:[0-9]*,(.*)/, "$1"); - setState(self.DONE); + self.status = self.responseText.replace( + /^NODE-XMLHTTPREQUEST-STATUS:([0-9]*),.*/, + '$1' + ) + self.responseText = self.responseText.replace( + /^NODE-XMLHTTPREQUEST-STATUS:[0-9]*,(.*)/, + '$1' + ) + setState(self.DONE) } } - }; + } /** * Called when an error is encountered to deal with it. */ - this.handleError = function(error) { - this.status = 503; - this.statusText = error; - this.responseText = error.stack; - errorFlag = true; - setState(this.DONE); - }; + this.handleError = function (error) { + this.status = 503 + this.statusText = error + this.responseText = error.stack + errorFlag = true + setState(this.DONE) + } /** * Aborts a request. */ - this.abort = function() { + this.abort = function () { if (request) { - request.abort(); - request = null; + request.abort() + request = null } - headers = defaultHeaders; - this.responseText = ""; - this.responseXML = ""; + headers = defaultHeaders + this.responseText = '' + this.responseXML = '' - errorFlag = true; + errorFlag = true - if (this.readyState !== this.UNSENT - && (this.readyState !== this.OPENED || sendFlag) - && this.readyState !== this.DONE) { - sendFlag = false; - setState(this.DONE); + if ( + this.readyState !== this.UNSENT && + (this.readyState !== this.OPENED || sendFlag) && + this.readyState !== this.DONE + ) { + sendFlag = false + setState(this.DONE) } - this.readyState = this.UNSENT; - }; + this.readyState = this.UNSENT + } /** * Adds an event listener. Preferred method of binding to events. */ - this.addEventListener = function(event, callback) { + this.addEventListener = function (event, callback) { if (!(event in listeners)) { - listeners[event] = []; + listeners[event] = [] } // Currently allows duplicate callbacks. Should it? - listeners[event].push(callback); - }; + listeners[event].push(callback) + } /** * Remove an event callback that has already been bound. * Only works on the matching funciton, cannot be a copy. */ - this.removeEventListener = function(event, callback) { + this.removeEventListener = function (event, callback) { if (event in listeners) { // Filter will return a new array with the callback removed - listeners[event] = listeners[event].filter(function(ev) { - return ev !== callback; - }); + listeners[event] = listeners[event].filter((ev) => { + return ev !== callback + }) } - }; + } /** * Dispatch any events, including both "on" methods and events attached using addEventListener. */ - this.dispatchEvent = function(event) { - if (typeof self["on" + event] === "function") { - self["on" + event](); + this.dispatchEvent = function (event) { + if (typeof self['on' + event] === 'function') { + self['on' + event]() } if (event in listeners) { for (var i = 0, len = listeners[event].length; i < len; i++) { - listeners[event][i].call(self); + listeners[event][i].call(self) } } - }; + } /** * Changes readyState and calls onreadystatechange. * * @param int state New state */ - var setState = function(state) { + var setState = function (state) { if (self.readyState !== state) { - self.readyState = state; + self.readyState = state - if (settings.async || self.readyState < self.OPENED || self.readyState === self.DONE) { - self.dispatchEvent("readystatechange"); + if ( + settings.async || + self.readyState < self.OPENED || + self.readyState === self.DONE + ) { + self.dispatchEvent('readystatechange') } if (self.readyState === self.DONE && !errorFlag) { - self.dispatchEvent("load"); + self.dispatchEvent('load') // @TODO figure out InspectorInstrumentation::didLoadXHR(cookie) - self.dispatchEvent("loadend"); + self.dispatchEvent('loadend') } } - }; -}; + } +} From b8b3fb8b115f75a8a70c55d72cf1acf5f21354d4 Mon Sep 17 00:00:00 2001 From: Jakob Ackermann Date: Wed, 24 Jun 2020 10:31:57 +0100 Subject: [PATCH 387/491] [misc] fix usage of deprecated node apis --- services/real-time/socket.io.patch.js | 18 ++++++++++++++++-- 1 file changed, 16 insertions(+), 2 deletions(-) diff --git a/services/real-time/socket.io.patch.js b/services/real-time/socket.io.patch.js index c4a0c051ec..3e655e24bb 100644 --- a/services/real-time/socket.io.patch.js +++ b/services/real-time/socket.io.patch.js @@ -1,6 +1,20 @@ // EventEmitter has been removed from process in node >= 7 // https://github.com/nodejs/node/commit/62b544290a075fe38e233887a06c408ba25a1c71 +/* + A socket.io dependency expects the EventEmitter to be available at + `process.EventEmitter`. + See this trace: + --- + + /app/node_modules/policyfile/lib/server.js:254 + Object.keys(process.EventEmitter.prototype).forEach(function proxy (key){ + ^ + + TypeError: Cannot read property 'prototype' of undefined + at Object. (/app/node_modules/policyfile/lib/server.js:254:34) + */ if (process.versions.node.split('.')[0] >= 7) { + // eslint-disable-next-line node/no-deprecated-api process.EventEmitter = require('events') } @@ -14,7 +28,7 @@ if (io.version === '0.9.16' || io.version === '0.9.19') { } function patchedFrameHandler(opcode, str) { - var dataBuffer = new Buffer(str) + var dataBuffer = Buffer.from(str) var dataLength = dataBuffer.length var startOffset = 2 var secondByte = dataLength @@ -29,7 +43,7 @@ function patchedFrameHandler(opcode, str) { startOffset = 4 secondByte = 126 } - var outputBuffer = new Buffer(dataLength + startOffset) + var outputBuffer = Buffer.alloc(dataLength + startOffset) outputBuffer[0] = opcode outputBuffer[1] = secondByte dataBuffer.copy(outputBuffer, startOffset) From c76bcb7732d555dc91e648e7ba4725ea4efb6ce6 Mon Sep 17 00:00:00 2001 From: Jakob Ackermann Date: Wed, 24 Jun 2020 10:40:08 +0100 Subject: [PATCH 388/491] [misc] fix eslint errors in XMLHttpRequest.js --- .../test/acceptance/libs/XMLHttpRequest.js | 42 ++++++++++--------- 1 file changed, 23 insertions(+), 19 deletions(-) diff --git a/services/real-time/test/acceptance/libs/XMLHttpRequest.js b/services/real-time/test/acceptance/libs/XMLHttpRequest.js index 21a60ad3bb..0222bc906b 100644 --- a/services/real-time/test/acceptance/libs/XMLHttpRequest.js +++ b/services/real-time/test/acceptance/libs/XMLHttpRequest.js @@ -11,7 +11,7 @@ * @license MIT */ -var Url = require('url') +const { URL } = require('url') var spawn = require('child_process').spawn var fs = require('fs') @@ -24,7 +24,6 @@ exports.XMLHttpRequest = function () { var https = require('https') // Holds http.js objects - var client var request var response @@ -147,8 +146,7 @@ exports.XMLHttpRequest = function () { // Check for valid request method if (!isAllowedHttpMethod(method)) { - throw 'SecurityError: Request method not allowed' - return + throw new Error('SecurityError: Request method not allowed') } settings = { @@ -169,15 +167,17 @@ exports.XMLHttpRequest = function () { * @param string value Header value */ this.setRequestHeader = function (header, value) { - if (this.readyState != this.OPENED) { - throw 'INVALID_STATE_ERR: setRequestHeader can only be called when state is OPEN' + if (this.readyState !== this.OPENED) { + throw new Error( + 'INVALID_STATE_ERR: setRequestHeader can only be called when state is OPEN' + ) } if (!isAllowedHttpHeader(header)) { console.warn('Refused to set unsafe header "' + header + '"') return } if (sendFlag) { - throw 'INVALID_STATE_ERR: send flag is true' + throw new Error('INVALID_STATE_ERR: send flag is true') } headers[header] = value } @@ -242,25 +242,29 @@ exports.XMLHttpRequest = function () { * @param string data Optional data to send as request body. */ this.send = function (data) { - if (this.readyState != this.OPENED) { - throw 'INVALID_STATE_ERR: connection must be opened before send() is called' + if (this.readyState !== this.OPENED) { + throw new Error( + 'INVALID_STATE_ERR: connection must be opened before send() is called' + ) } if (sendFlag) { - throw 'INVALID_STATE_ERR: send has already been called' + throw new Error('INVALID_STATE_ERR: send has already been called') } + var host var ssl = false var local = false - var url = Url.parse(settings.url) + var url = new URL(settings.url) // Determine the server switch (url.protocol) { case 'https:': ssl = true - // SSL & non-SSL both need host, no break here. + host = url.hostname + break case 'http:': - var host = url.hostname + host = url.hostname break case 'file:': @@ -269,17 +273,17 @@ exports.XMLHttpRequest = function () { case undefined: case '': - var host = 'localhost' + host = 'localhost' break default: - throw 'Protocol not supported.' + throw new Error('Protocol not supported.') } // Load files off the local filesystem (file://) if (local) { if (settings.method !== 'GET') { - throw 'XMLHttpRequest: Only GET method is supported' + throw new Error('XMLHttpRequest: Only GET method is supported') } if (settings.async) { @@ -322,7 +326,7 @@ exports.XMLHttpRequest = function () { if (typeof settings.password === 'undefined') { settings.password = '' } - var authBuf = new Buffer(settings.user + ':' + settings.password) + var authBuf = Buffer.from(settings.user + ':' + settings.password) headers.Authorization = 'Basic ' + authBuf.toString('base64') } @@ -443,8 +447,8 @@ exports.XMLHttpRequest = function () { (data ? "req.write('" + data.replace(/'/g, "\\'") + "');" : '') + 'req.end();' // Start the other Node Process, executing this string - syncProc = spawn(process.argv[0], ['-e', execString]) - while ((self.responseText = fs.readFileSync(syncFile, 'utf8')) == '') { + const syncProc = spawn(process.argv[0], ['-e', execString]) + while ((self.responseText = fs.readFileSync(syncFile, 'utf8')) === '') { // Wait while the file is empty } // Kill the child process once the file has data From 9c7bfe020c0648a165bdbdf5d6f45f3ce09ca411 Mon Sep 17 00:00:00 2001 From: Jakob Ackermann Date: Wed, 24 Jun 2020 10:41:55 +0100 Subject: [PATCH 389/491] [misc] fix eslint errors in app.js --- services/real-time/app.js | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/services/real-time/app.js b/services/real-time/app.js index 47cae86f45..b34b33df61 100644 --- a/services/real-time/app.js +++ b/services/real-time/app.js @@ -9,7 +9,6 @@ const Metrics = require('metrics-sharelatex') const Settings = require('settings-sharelatex') Metrics.initialize(Settings.appName || 'real-time') const async = require('async') -const _ = require('underscore') const logger = require('logger-sharelatex') logger.initialize('real-time') @@ -214,7 +213,7 @@ if (Settings.shutdownDrainTimeWindow != null) { } if (Settings.continualPubsubTraffic) { - console.log('continualPubsubTraffic enabled') + logger.warn('continualPubsubTraffic enabled') const pubsubClient = redis.createClient(Settings.redis.pubsub) const clusterClient = redis.createClient(Settings.redis.websessions) @@ -239,7 +238,7 @@ if (Settings.continualPubsubTraffic) { } var runPubSubTraffic = () => - async.map(['applied-ops', 'editor-events'], publishJob, (err) => + async.map(['applied-ops', 'editor-events'], publishJob, () => setTimeout(runPubSubTraffic, 1000 * 20) ) From fa42166be3dad003ce85a893e4b720f0fd85e199 Mon Sep 17 00:00:00 2001 From: Jakob Ackermann Date: Wed, 24 Jun 2020 10:44:38 +0100 Subject: [PATCH 390/491] [misc] ignore camelcase warning in settings (key_schema parameter) --- services/real-time/config/settings.defaults.js | 2 ++ 1 file changed, 2 insertions(+) diff --git a/services/real-time/config/settings.defaults.js b/services/real-time/config/settings.defaults.js index 8f3d562f8b..1e4ec6fe72 100644 --- a/services/real-time/config/settings.defaults.js +++ b/services/real-time/config/settings.defaults.js @@ -1,3 +1,5 @@ +/* eslint-disable camelcase */ + const settings = { redis: { pubsub: { From 5e191be171d40d3dfe7e8c5f9b346cd133dfa316 Mon Sep 17 00:00:00 2001 From: Jakob Ackermann Date: Wed, 24 Jun 2020 11:06:57 +0100 Subject: [PATCH 391/491] [misc] replace console logging with logger-sharelatex --- services/real-time/socket.io.patch.js | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/services/real-time/socket.io.patch.js b/services/real-time/socket.io.patch.js index 3e655e24bb..5852a4f266 100644 --- a/services/real-time/socket.io.patch.js +++ b/services/real-time/socket.io.patch.js @@ -19,9 +19,10 @@ if (process.versions.node.split('.')[0] >= 7) { } var io = require('socket.io') +const logger = require('logger-sharelatex') if (io.version === '0.9.16' || io.version === '0.9.19') { - console.log('patching socket.io hybi-16 transport frame prototype') + logger.warn('patching socket.io hybi-16 transport frame prototype') var transports = require('socket.io/lib/transports/websocket/hybi-16.js') transports.prototype.frame = patchedFrameHandler // file hybi-07-12 has the same problem but no browsers are using that protocol now @@ -33,7 +34,7 @@ function patchedFrameHandler(opcode, str) { var startOffset = 2 var secondByte = dataLength if (dataLength === 65536) { - console.log('fixing invalid frame length in socket.io') + logger.log('fixing invalid frame length in socket.io') } if (dataLength > 65535) { // original code had > 65536 From bea0e9beb7d2af48c9757aaa4b399077cb51d164 Mon Sep 17 00:00:00 2001 From: Jakob Ackermann Date: Mon, 29 Jun 2020 11:57:16 +0100 Subject: [PATCH 392/491] [misc] RoomManagerTests: explicitly pass a cb to joinProject/joinDoc --- services/real-time/test/unit/js/RoomManagerTests.js | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/services/real-time/test/unit/js/RoomManagerTests.js b/services/real-time/test/unit/js/RoomManagerTests.js index 3aee509af5..a202720ec7 100644 --- a/services/real-time/test/unit/js/RoomManagerTests.js +++ b/services/real-time/test/unit/js/RoomManagerTests.js @@ -113,7 +113,7 @@ describe('RoomManager', function () { }) return describe('when there are other clients in the project room', function () { - beforeEach(function () { + beforeEach(function (done) { this.RoomManager._clientsInRoom .withArgs(this.client, this.project_id) .onFirstCall() @@ -121,7 +121,7 @@ describe('RoomManager', function () { .onSecondCall() .returns(124) this.client.join = sinon.stub() - return this.RoomManager.joinProject(this.client, this.project_id) + this.RoomManager.joinProject(this.client, this.project_id, done) }) it('should join the room using the id', function () { @@ -174,7 +174,7 @@ describe('RoomManager', function () { }) return describe('when there are other clients in the doc room', function () { - beforeEach(function () { + beforeEach(function (done) { this.RoomManager._clientsInRoom .withArgs(this.client, this.doc_id) .onFirstCall() @@ -182,7 +182,7 @@ describe('RoomManager', function () { .onSecondCall() .returns(124) this.client.join = sinon.stub() - return this.RoomManager.joinDoc(this.client, this.doc_id) + this.RoomManager.joinDoc(this.client, this.doc_id, done) }) it('should join the room using the id', function () { From 4d01c02946c9f8372b822af54c53bef29f084fa6 Mon Sep 17 00:00:00 2001 From: Jakob Ackermann Date: Mon, 29 Jun 2020 17:15:29 +0100 Subject: [PATCH 393/491] [misc] WebsocketControllerTests: explicitly pass a cb to leaveProject ...and fix the stubbing of `io.sockets.clients`. We were running the assertions prior to the actual completion of the leaveProject. Which in turn shadowed the broken test setup -- `io.sockets.clients()` was returning the `this.clientsInRoom=[]` from a previous context. --- .../test/unit/js/WebsocketControllerTests.js | 15 +++++++++++++-- 1 file changed, 13 insertions(+), 2 deletions(-) diff --git a/services/real-time/test/unit/js/WebsocketControllerTests.js b/services/real-time/test/unit/js/WebsocketControllerTests.js index 8d8a39bb74..3cf4100951 100644 --- a/services/real-time/test/unit/js/WebsocketControllerTests.js +++ b/services/real-time/test/unit/js/WebsocketControllerTests.js @@ -42,6 +42,7 @@ describe('WebsocketController', function () { leave: sinon.stub() } return (this.WebsocketController = SandboxedModule.require(modulePath, { + globals: { console }, requires: { './WebApiManager': (this.WebApiManager = {}), './AuthorizationManager': (this.AuthorizationManager = {}), @@ -393,9 +394,19 @@ describe('WebsocketController', function () { }) describe('when the project is not empty', function () { - beforeEach(function () { + beforeEach(function (done) { this.clientsInRoom = ['mock-remaining-client'] - return this.WebsocketController.leaveProject(this.io, this.client) + this.io = { + sockets: { + clients: (room_id) => { + if (room_id !== this.project_id) { + throw 'expected room_id to be project_id' + } + return this.clientsInRoom + } + } + } + return this.WebsocketController.leaveProject(this.io, this.client, done) }) return it('should not flush the project in the document updater', function () { From a77222470bd3f4502700ba780fd18ccfcc7e9d53 Mon Sep 17 00:00:00 2001 From: Jakob Ackermann Date: Mon, 29 Jun 2020 18:11:33 +0100 Subject: [PATCH 394/491] [misc] WebsocketControllerTests: always pass cb to updateClientPosition ...and fix the inconsistent async behaviour for logged-in vs anonymous users in the app. --- .../real-time/app/js/WebsocketController.js | 3 +- .../test/unit/js/WebsocketControllerTests.js | 63 ++++++++++++------- 2 files changed, 44 insertions(+), 22 deletions(-) diff --git a/services/real-time/app/js/WebsocketController.js b/services/real-time/app/js/WebsocketController.js index 92632cc154..451c514336 100644 --- a/services/real-time/app/js/WebsocketController.js +++ b/services/real-time/app/js/WebsocketController.js @@ -380,7 +380,8 @@ module.exports = WebsocketController = { // Don't store anonymous users in redis to avoid influx if (!user_id || user_id === 'anonymous-user') { cursorData.name = '' - callback() + // consistent async behaviour + setTimeout(callback) } else { cursorData.name = first_name && last_name diff --git a/services/real-time/test/unit/js/WebsocketControllerTests.js b/services/real-time/test/unit/js/WebsocketControllerTests.js index 3cf4100951..b6792a1cc2 100644 --- a/services/real-time/test/unit/js/WebsocketControllerTests.js +++ b/services/real-time/test/unit/js/WebsocketControllerTests.js @@ -78,7 +78,9 @@ describe('WebsocketController', function () { } } this.privilegeLevel = 'owner' - this.ConnectedUsersManager.updateUserPosition = sinon.stub().callsArg(4) + this.ConnectedUsersManager.updateUserPosition = sinon + .stub() + .callsArgAsync(4) this.isRestrictedUser = true this.WebApiManager.joinProject = sinon .stub() @@ -213,7 +215,9 @@ describe('WebsocketController', function () { } } this.privilegeLevel = 'owner' - this.ConnectedUsersManager.updateUserPosition = sinon.stub().callsArg(4) + this.ConnectedUsersManager.updateUserPosition = sinon + .stub() + .callsArgAsync(4) this.isRestrictedUser = true this.WebApiManager.joinProject = sinon .stub() @@ -964,7 +968,7 @@ describe('WebsocketController', function () { this.WebsocketLoadBalancer.emitToRoom = sinon.stub() this.ConnectedUsersManager.updateUserPosition = sinon .stub() - .callsArgWith(4) + .callsArgAsync(4) this.AuthorizationManager.assertClientCanViewProjectAndDoc = sinon .stub() .callsArgWith(2, null) @@ -976,7 +980,7 @@ describe('WebsocketController', function () { }) describe('with a logged in user', function () { - beforeEach(function () { + beforeEach(function (done) { this.client.ol_context = { project_id: this.project_id, first_name: (this.first_name = 'Douglas'), @@ -984,9 +988,8 @@ describe('WebsocketController', function () { email: (this.email = 'joe@example.com'), user_id: (this.user_id = 'user-id-123') } - this.WebsocketController.updateClientPosition(this.client, this.update) - return (this.populatedCursorData = { + this.populatedCursorData = { doc_id: this.doc_id, id: this.client.publicId, name: `${this.first_name} ${this.last_name}`, @@ -994,7 +997,12 @@ describe('WebsocketController', function () { column: this.column, email: this.email, user_id: this.user_id - }) + } + this.WebsocketController.updateClientPosition( + this.client, + this.update, + done + ) }) it("should send the update to the project room with the user's name", function () { @@ -1036,7 +1044,7 @@ describe('WebsocketController', function () { }) describe('with a logged in user who has no last_name set', function () { - beforeEach(function () { + beforeEach(function (done) { this.client.ol_context = { project_id: this.project_id, first_name: (this.first_name = 'Douglas'), @@ -1044,9 +1052,8 @@ describe('WebsocketController', function () { email: (this.email = 'joe@example.com'), user_id: (this.user_id = 'user-id-123') } - this.WebsocketController.updateClientPosition(this.client, this.update) - return (this.populatedCursorData = { + this.populatedCursorData = { doc_id: this.doc_id, id: this.client.publicId, name: `${this.first_name}`, @@ -1054,7 +1061,12 @@ describe('WebsocketController', function () { column: this.column, email: this.email, user_id: this.user_id - }) + } + this.WebsocketController.updateClientPosition( + this.client, + this.update, + done + ) }) it("should send the update to the project room with the user's name", function () { @@ -1096,7 +1108,7 @@ describe('WebsocketController', function () { }) describe('with a logged in user who has no first_name set', function () { - beforeEach(function () { + beforeEach(function (done) { this.client.ol_context = { project_id: this.project_id, first_name: undefined, @@ -1104,9 +1116,8 @@ describe('WebsocketController', function () { email: (this.email = 'joe@example.com'), user_id: (this.user_id = 'user-id-123') } - this.WebsocketController.updateClientPosition(this.client, this.update) - return (this.populatedCursorData = { + this.populatedCursorData = { doc_id: this.doc_id, id: this.client.publicId, name: `${this.last_name}`, @@ -1114,7 +1125,12 @@ describe('WebsocketController', function () { column: this.column, email: this.email, user_id: this.user_id - }) + } + this.WebsocketController.updateClientPosition( + this.client, + this.update, + done + ) }) it("should send the update to the project room with the user's name", function () { @@ -1155,7 +1171,7 @@ describe('WebsocketController', function () { }) }) describe('with a logged in user who has no names set', function () { - beforeEach(function () { + beforeEach(function (done) { this.client.ol_context = { project_id: this.project_id, first_name: undefined, @@ -1165,7 +1181,8 @@ describe('WebsocketController', function () { } return this.WebsocketController.updateClientPosition( this.client, - this.update + this.update, + done ) }) @@ -1185,13 +1202,14 @@ describe('WebsocketController', function () { }) describe('with an anonymous user', function () { - beforeEach(function () { + beforeEach(function (done) { this.client.ol_context = { project_id: this.project_id } return this.WebsocketController.updateClientPosition( this.client, - this.update + this.update, + done ) }) @@ -1214,13 +1232,16 @@ describe('WebsocketController', function () { }) return describe('when the client has disconnected', function () { - beforeEach(function () { + beforeEach(function (done) { this.client.disconnected = true this.AuthorizationManager.assertClientCanViewProjectAndDoc = sinon.stub() return this.WebsocketController.updateClientPosition( this.client, this.update, - this.callback + (...args) => { + this.callback(...args) + done(args[0]) + } ) }) From 8601084a2eb89f4863272db116f8ef5febb80e11 Mon Sep 17 00:00:00 2001 From: Jakob Ackermann Date: Wed, 1 Jul 2020 11:36:36 +0100 Subject: [PATCH 395/491] [misc] WebsocketControllerTests: remove debugging code --- services/real-time/test/unit/js/WebsocketControllerTests.js | 1 - 1 file changed, 1 deletion(-) diff --git a/services/real-time/test/unit/js/WebsocketControllerTests.js b/services/real-time/test/unit/js/WebsocketControllerTests.js index b6792a1cc2..7796ca7275 100644 --- a/services/real-time/test/unit/js/WebsocketControllerTests.js +++ b/services/real-time/test/unit/js/WebsocketControllerTests.js @@ -42,7 +42,6 @@ describe('WebsocketController', function () { leave: sinon.stub() } return (this.WebsocketController = SandboxedModule.require(modulePath, { - globals: { console }, requires: { './WebApiManager': (this.WebApiManager = {}), './AuthorizationManager': (this.AuthorizationManager = {}), From aa9d6c8dc95f3032e920890916e59c611ede8f22 Mon Sep 17 00:00:00 2001 From: Jakob Ackermann Date: Tue, 7 Jul 2020 12:06:02 +0200 Subject: [PATCH 396/491] [misc] reland decaff cleanup (#166) * [misc] decaff cleanup: RoomManager * [misc] decaff cleanup: RedisClientManager * [misc] decaff cleanup: SafeJsonParse * [misc] decaff cleanup: WebApiManager * [misc] decaff cleanup: WebsocketController * [misc] decaff cleanup: WebsocketLoadBalancer * [misc] decaff cleanup: SessionSockets * [misc] decaff cleanup: HttpController * [misc] decaff cleanup: HttpApiController * [misc] decaff cleanup: HealthCheckManager * [misc] decaff cleanup: EventLogger * [misc] decaff cleanup: Errors o-error will eliminate most of it -- when we migrate over. * [misc] decaff cleanup: DrainManager * [misc] decaff cleanup: DocumentUpdaterManager * [misc] decaff cleanup: DocumentUpdaterController: no-unused-vars * [misc] decaff cleanup: DocumentUpdaterController: Array.from * [misc] decaff cleanup: DocumentUpdaterController: implicit return * [misc] decaff cleanup: DocumentUpdaterController: IIFE * [misc] decaff cleanup: DocumentUpdaterController: null checks * [misc] decaff cleanup: DocumentUpdaterController: simpler loops * [misc] decaff cleanup: DocumentUpdaterController: move module name def * [misc] decaff cleanup: ConnectedUsersManager: handle-callback-err * [misc] decaff cleanup: ConnectedUsersManager: implicit returns * [misc] decaff cleanup: ConnectedUsersManager: null checks * [misc] decaff cleanup: ChannelManager: no-unused-vars * [misc] decaff cleanup: ChannelManager: implicit returns * [misc] decaff cleanup: ChannelManager: other cleanup - var -> const - drop variable assignment before return * [misc] decaff cleanup: AuthorizationManager: handle-callback-err Note: This requires a change in WebsocketController to provide a dummy callback. * [misc] decaff cleanup: AuthorizationManager: Array.from * [misc] decaff cleanup: AuthorizationManager: implicit returns * [misc] decaff cleanup: AuthorizationManager: null checks * [misc] decaff cleanup: Router: handle-callback-err * [misc] decaff cleanup: Router: standard/no-callback-literal * [misc] decaff cleanup: Router: Array.from * [misc] decaff cleanup: Router: implicit returns * [misc] decaff cleanup: Router: refactor __guard__ wrapper * [misc] decaff cleanup: Router: null checks And a minor bug fix: user.id -> user._id * [misc] decaff cleanup: Router: move variable declarations to assignments * [misc] decaff cleanup: app: implicit returns * [misc] decaff cleanup: app: __guard__ * [misc] decaff cleanup: app: null checks * [misc] decaff cleanup: app: function definitions * [misc] decaff cleanup: app: drop unused next argument * [misc] decaff cleanup: app: var -> const --- services/real-time/app.js | 86 +++---- .../real-time/app/js/AuthorizationManager.js | 76 ++---- services/real-time/app/js/ChannelManager.js | 37 ++- .../real-time/app/js/ConnectedUsersManager.js | 112 ++++----- .../app/js/DocumentUpdaterController.js | 106 +++------ .../app/js/DocumentUpdaterManager.js | 60 ++--- services/real-time/app/js/DrainManager.js | 34 +-- services/real-time/app/js/Errors.js | 22 +- services/real-time/app/js/EventLogger.js | 34 +-- .../real-time/app/js/HealthCheckManager.js | 38 +-- .../real-time/app/js/HttpApiController.js | 26 +-- services/real-time/app/js/HttpController.js | 58 ++--- .../real-time/app/js/RedisClientManager.js | 45 +--- services/real-time/app/js/RoomManager.js | 115 ++++------ services/real-time/app/js/Router.js | 216 +++++++---------- services/real-time/app/js/SafeJsonParse.js | 18 +- services/real-time/app/js/SessionSockets.js | 16 +- services/real-time/app/js/WebApiManager.js | 35 +-- .../real-time/app/js/WebsocketController.js | 217 +++++++----------- .../real-time/app/js/WebsocketLoadBalancer.js | 115 +++------- 20 files changed, 483 insertions(+), 983 deletions(-) diff --git a/services/real-time/app.js b/services/real-time/app.js index b34b33df61..1580f18027 100644 --- a/services/real-time/app.js +++ b/services/real-time/app.js @@ -1,10 +1,3 @@ -/* - * decaffeinate suggestions: - * DS102: Remove unnecessary code created because of implicit returns - * DS103: Rewrite code to no longer use __guard__ - * DS207: Consider shorter variations of null checks - * Full docs: https://github.com/decaffeinate/decaffeinate/blob/master/docs/suggestions.md - */ const Metrics = require('metrics-sharelatex') const Settings = require('settings-sharelatex') Metrics.initialize(Settings.appName || 'real-time') @@ -17,7 +10,7 @@ Metrics.event_loop.monitor(logger) const express = require('express') const session = require('express-session') const redis = require('redis-sharelatex') -if ((Settings.sentry != null ? Settings.sentry.dsn : undefined) != null) { +if (Settings.sentry && Settings.sentry.dsn) { logger.initializeErrorReporting(Settings.sentry.dsn) } @@ -70,44 +63,43 @@ io.configure(function () { 'xhr-polling', 'jsonp-polling' ]) - return io.set('log level', 1) + io.set('log level', 1) }) -app.get('/', (req, res, next) => res.send('real-time-sharelatex is alive')) +app.get('/', (req, res) => res.send('real-time-sharelatex is alive')) -app.get('/status', function (req, res, next) { +app.get('/status', function (req, res) { if (Settings.shutDownInProgress) { - return res.send(503) // Service unavailable + res.send(503) // Service unavailable } else { - return res.send('real-time-sharelatex is alive') + res.send('real-time-sharelatex is alive') } }) -app.get('/debug/events', function (req, res, next) { - Settings.debugEvents = - parseInt(req.query != null ? req.query.count : undefined, 10) || 20 +app.get('/debug/events', function (req, res) { + Settings.debugEvents = parseInt(req.query.count, 10) || 20 logger.log({ count: Settings.debugEvents }, 'starting debug mode') - return res.send(`debug mode will log next ${Settings.debugEvents} events`) + res.send(`debug mode will log next ${Settings.debugEvents} events`) }) const rclient = require('redis-sharelatex').createClient( Settings.redis.realtime ) -const healthCheck = (req, res, next) => +function healthCheck(req, res) { rclient.healthCheck(function (error) { - if (error != null) { + if (error) { logger.err({ err: error }, 'failed redis health check') - return res.sendStatus(500) + res.sendStatus(500) } else if (HealthCheckManager.isFailing()) { const status = HealthCheckManager.status() logger.err({ pubSubErrors: status }, 'failed pubsub health check') - return res.sendStatus(500) + res.sendStatus(500) } else { - return res.sendStatus(200) + res.sendStatus(200) } }) - +} app.get('/health_check', healthCheck) app.get('/health_check/redis', healthCheck) @@ -125,30 +117,30 @@ const { port } = Settings.internal.realTime const { host } = Settings.internal.realTime server.listen(port, host, function (error) { - if (error != null) { + if (error) { throw error } - return logger.info(`realtime starting up, listening on ${host}:${port}`) + logger.info(`realtime starting up, listening on ${host}:${port}`) }) // Stop huge stack traces in logs from all the socket.io parsing steps. Error.stackTraceLimit = 10 -var shutdownCleanly = function (signal) { - const connectedClients = __guard__(io.sockets.clients(), (x) => x.length) +function shutdownCleanly(signal) { + const connectedClients = io.sockets.clients().length if (connectedClients === 0) { logger.warn('no clients connected, exiting') - return process.exit() + process.exit() } else { logger.warn( { connectedClients }, 'clients still connected, not shutting down yet' ) - return setTimeout(() => shutdownCleanly(signal), 30 * 1000) + setTimeout(() => shutdownCleanly(signal), 30 * 1000) } } -const drainAndShutdown = function (signal) { +function drainAndShutdown(signal) { if (Settings.shutDownInProgress) { logger.warn({ signal }, 'shutdown already in progress, ignoring signal') } else { @@ -160,20 +152,20 @@ const drainAndShutdown = function (signal) { `received interrupt, delay drain by ${statusCheckInterval}ms` ) } - return setTimeout(function () { + setTimeout(function () { logger.warn( { signal }, `received interrupt, starting drain over ${shutdownDrainTimeWindow} mins` ) DrainManager.startDrainTimeWindow(io, shutdownDrainTimeWindow) - return shutdownCleanly(signal) + shutdownCleanly(signal) }, statusCheckInterval) } } Settings.shutDownInProgress = false -if (Settings.shutdownDrainTimeWindow != null) { - var shutdownDrainTimeWindow = parseInt(Settings.shutdownDrainTimeWindow, 10) +const shutdownDrainTimeWindow = parseInt(Settings.shutdownDrainTimeWindow, 10) +if (Settings.shutdownDrainTimeWindow) { logger.log({ shutdownDrainTimeWindow }, 'shutdownDrainTimeWindow enabled') for (const signal of [ 'SIGINT', @@ -188,9 +180,7 @@ if (Settings.shutdownDrainTimeWindow != null) { } // signal is passed as argument to event handler // global exception handler - if ( - Settings.errors != null ? Settings.errors.catchUncaughtErrors : undefined - ) { + if (Settings.errors && Settings.errors.catchUncaughtErrors) { process.removeAllListeners('uncaughtException') process.on('uncaughtException', function (error) { if (['EPIPE', 'ECONNRESET'].includes(error.code)) { @@ -201,12 +191,8 @@ if (Settings.shutdownDrainTimeWindow != null) { ) } logger.error({ err: error }, 'uncaught exception') - if ( - Settings.errors != null - ? Settings.errors.shutdownOnUncaughtError - : undefined - ) { - return drainAndShutdown('SIGABRT') + if (Settings.errors && Settings.errors.shutdownOnUncaughtError) { + drainAndShutdown('SIGABRT') } }) } @@ -227,26 +213,20 @@ if (Settings.continualPubsubTraffic) { date: new Date().toString() }) Metrics.summary(`redis.publish.${channel}`, json.length) - return pubsubClient.publish(channel, json, function (err) { - if (err != null) { + pubsubClient.publish(channel, json, function (err) { + if (err) { logger.err({ err, channel }, 'error publishing pubsub traffic to redis') } const blob = JSON.stringify({ keep: 'alive' }) Metrics.summary('redis.publish.cluster-continual-traffic', blob.length) - return clusterClient.publish('cluster-continual-traffic', blob, callback) + clusterClient.publish('cluster-continual-traffic', blob, callback) }) } - var runPubSubTraffic = () => + const runPubSubTraffic = () => async.map(['applied-ops', 'editor-events'], publishJob, () => setTimeout(runPubSubTraffic, 1000 * 20) ) runPubSubTraffic() } - -function __guard__(value, transform) { - return typeof value !== 'undefined' && value !== null - ? transform(value) - : undefined -} diff --git a/services/real-time/app/js/AuthorizationManager.js b/services/real-time/app/js/AuthorizationManager.js index 15607a898a..a545594479 100644 --- a/services/real-time/app/js/AuthorizationManager.js +++ b/services/real-time/app/js/AuthorizationManager.js @@ -1,23 +1,10 @@ /* eslint-disable camelcase, - handle-callback-err, */ -// TODO: This file was created by bulk-decaffeinate. -// Fix any style issues and re-enable lint. -/* - * decaffeinate suggestions: - * DS101: Remove unnecessary use of Array.from - * DS102: Remove unnecessary code created because of implicit returns - * DS207: Consider shorter variations of null checks - * Full docs: https://github.com/decaffeinate/decaffeinate/blob/master/docs/suggestions.md - */ let AuthorizationManager module.exports = AuthorizationManager = { assertClientCanViewProject(client, callback) { - if (callback == null) { - callback = function (error) {} - } - return AuthorizationManager._assertClientHasPrivilegeLevel( + AuthorizationManager._assertClientHasPrivilegeLevel( client, ['readOnly', 'readAndWrite', 'owner'], callback @@ -25,10 +12,7 @@ module.exports = AuthorizationManager = { }, assertClientCanEditProject(client, callback) { - if (callback == null) { - callback = function (error) {} - } - return AuthorizationManager._assertClientHasPrivilegeLevel( + AuthorizationManager._assertClientHasPrivilegeLevel( client, ['readAndWrite', 'owner'], callback @@ -36,76 +20,46 @@ module.exports = AuthorizationManager = { }, _assertClientHasPrivilegeLevel(client, allowedLevels, callback) { - if (callback == null) { - callback = function (error) {} - } - if (Array.from(allowedLevels).includes(client.ol_context.privilege_level)) { - return callback(null) + if (allowedLevels.includes(client.ol_context.privilege_level)) { + callback(null) } else { - return callback(new Error('not authorized')) + callback(new Error('not authorized')) } }, assertClientCanViewProjectAndDoc(client, doc_id, callback) { - if (callback == null) { - callback = function (error) {} - } - return AuthorizationManager.assertClientCanViewProject(client, function ( - error - ) { - if (error != null) { + AuthorizationManager.assertClientCanViewProject(client, function (error) { + if (error) { return callback(error) } - return AuthorizationManager._assertClientCanAccessDoc( - client, - doc_id, - callback - ) + AuthorizationManager._assertClientCanAccessDoc(client, doc_id, callback) }) }, assertClientCanEditProjectAndDoc(client, doc_id, callback) { - if (callback == null) { - callback = function (error) {} - } - return AuthorizationManager.assertClientCanEditProject(client, function ( - error - ) { - if (error != null) { + AuthorizationManager.assertClientCanEditProject(client, function (error) { + if (error) { return callback(error) } - return AuthorizationManager._assertClientCanAccessDoc( - client, - doc_id, - callback - ) + AuthorizationManager._assertClientCanAccessDoc(client, doc_id, callback) }) }, _assertClientCanAccessDoc(client, doc_id, callback) { - if (callback == null) { - callback = function (error) {} - } if (client.ol_context[`doc:${doc_id}`] === 'allowed') { - return callback(null) + callback(null) } else { - return callback(new Error('not authorized')) + callback(new Error('not authorized')) } }, addAccessToDoc(client, doc_id, callback) { - if (callback == null) { - callback = function (error) {} - } client.ol_context[`doc:${doc_id}`] = 'allowed' - return callback(null) + callback(null) }, removeAccessToDoc(client, doc_id, callback) { - if (callback == null) { - callback = function (error) {} - } delete client.ol_context[`doc:${doc_id}`] - return callback(null) + callback(null) } } diff --git a/services/real-time/app/js/ChannelManager.js b/services/real-time/app/js/ChannelManager.js index 09e81cebf5..e84cfe44a9 100644 --- a/services/real-time/app/js/ChannelManager.js +++ b/services/real-time/app/js/ChannelManager.js @@ -1,14 +1,3 @@ -/* eslint-disable - no-unused-vars, -*/ -// TODO: This file was created by bulk-decaffeinate. -// Fix any style issues and re-enable lint. -/* - * decaffeinate suggestions: - * DS102: Remove unnecessary code created because of implicit returns - * Full docs: https://github.com/decaffeinate/decaffeinate/blob/master/docs/suggestions.md - */ -let ChannelManager const logger = require('logger-sharelatex') const metrics = require('metrics-sharelatex') const settings = require('settings-sharelatex') @@ -19,7 +8,7 @@ const ClientMap = new Map() // for each redis client, store a Map of subscribed // that we never subscribe to a channel multiple times. The socket.io side is // handled by RoomManager. -module.exports = ChannelManager = { +module.exports = { getClientMapEntry(rclient) { // return the per-client channel map if it exists, otherwise create and // return an empty map for the client. @@ -36,22 +25,25 @@ module.exports = ChannelManager = { const p = rclient.subscribe(channel) p.finally(function () { if (clientChannelMap.get(channel) === subscribePromise) { - return clientChannelMap.delete(channel) + clientChannelMap.delete(channel) } }) .then(function () { logger.log({ channel }, 'subscribed to channel') - return metrics.inc(`subscribe.${baseChannel}`) + metrics.inc(`subscribe.${baseChannel}`) }) .catch(function (err) { logger.error({ channel, err }, 'failed to subscribe to channel') - return metrics.inc(`subscribe.failed.${baseChannel}`) + metrics.inc(`subscribe.failed.${baseChannel}`) }) return p } const pendingActions = clientChannelMap.get(channel) || Promise.resolve() - var subscribePromise = pendingActions.then(actualSubscribe, actualSubscribe) + const subscribePromise = pendingActions.then( + actualSubscribe, + actualSubscribe + ) clientChannelMap.set(channel, subscribePromise) logger.log({ channel }, 'planned to subscribe to channel') return subscribePromise @@ -62,26 +54,25 @@ module.exports = ChannelManager = { const channel = `${baseChannel}:${id}` const actualUnsubscribe = function () { // unsubscribe is happening in the background, it should not reject - const p = rclient + return rclient .unsubscribe(channel) .finally(function () { if (clientChannelMap.get(channel) === unsubscribePromise) { - return clientChannelMap.delete(channel) + clientChannelMap.delete(channel) } }) .then(function () { logger.log({ channel }, 'unsubscribed from channel') - return metrics.inc(`unsubscribe.${baseChannel}`) + metrics.inc(`unsubscribe.${baseChannel}`) }) .catch(function (err) { logger.error({ channel, err }, 'unsubscribed from channel') - return metrics.inc(`unsubscribe.failed.${baseChannel}`) + metrics.inc(`unsubscribe.failed.${baseChannel}`) }) - return p } const pendingActions = clientChannelMap.get(channel) || Promise.resolve() - var unsubscribePromise = pendingActions.then( + const unsubscribePromise = pendingActions.then( actualUnsubscribe, actualUnsubscribe ) @@ -100,6 +91,6 @@ module.exports = ChannelManager = { } // we publish on a different client to the subscribe, so we can't // check for the channel existing here - return rclient.publish(channel, data) + rclient.publish(channel, data) } } diff --git a/services/real-time/app/js/ConnectedUsersManager.js b/services/real-time/app/js/ConnectedUsersManager.js index 6770dd5421..06ea442d63 100644 --- a/services/real-time/app/js/ConnectedUsersManager.js +++ b/services/real-time/app/js/ConnectedUsersManager.js @@ -1,15 +1,6 @@ /* eslint-disable camelcase, - handle-callback-err, */ -// TODO: This file was created by bulk-decaffeinate. -// Fix any style issues and re-enable lint. -/* - * decaffeinate suggestions: - * DS102: Remove unnecessary code created because of implicit returns - * DS207: Consider shorter variations of null checks - * Full docs: https://github.com/decaffeinate/decaffeinate/blob/master/docs/suggestions.md - */ const async = require('async') const Settings = require('settings-sharelatex') const logger = require('logger-sharelatex') @@ -29,9 +20,6 @@ module.exports = { // update. This way we don't care if the connected_user key has expired when // we receive a cursor update. updateUserPosition(project_id, client_id, user, cursorData, callback) { - if (callback == null) { - callback = function (err) {} - } logger.log({ project_id, client_id }, 'marking user as joined or connected') const multi = rclient.multi() @@ -65,7 +53,7 @@ module.exports = { user.email || '' ) - if (cursorData != null) { + if (cursorData) { multi.hset( Keys.connectedUser({ project_id, client_id }), 'cursorData', @@ -77,21 +65,18 @@ module.exports = { USER_TIMEOUT_IN_S ) - return multi.exec(function (err) { - if (err != null) { + multi.exec(function (err) { + if (err) { logger.err( { err, project_id, client_id }, 'problem marking user as connected' ) } - return callback(err) + callback(err) }) }, - refreshClient(project_id, client_id, callback) { - if (callback == null) { - callback = function (err) {} - } + refreshClient(project_id, client_id) { logger.log({ project_id, client_id }, 'refreshing connected client') const multi = rclient.multi() multi.hset( @@ -103,14 +88,13 @@ module.exports = { Keys.connectedUser({ project_id, client_id }), USER_TIMEOUT_IN_S ) - return multi.exec(function (err) { - if (err != null) { + multi.exec(function (err) { + if (err) { logger.err( { err, project_id, client_id }, 'problem refreshing connected client' ) } - return callback(err) }) }, @@ -120,74 +104,66 @@ module.exports = { multi.srem(Keys.clientsInProject({ project_id }), client_id) multi.expire(Keys.clientsInProject({ project_id }), FOUR_DAYS_IN_S) multi.del(Keys.connectedUser({ project_id, client_id })) - return multi.exec(callback) + multi.exec(callback) }, _getConnectedUser(project_id, client_id, callback) { - return rclient.hgetall( - Keys.connectedUser({ project_id, client_id }), - function (err, result) { - if ( - result == null || - Object.keys(result).length === 0 || - !result.user_id - ) { - result = { - connected: false, - client_id - } - } else { - result.connected = true - result.client_id = client_id - result.client_age = - (Date.now() - parseInt(result.last_updated_at, 10)) / 1000 - if (result.cursorData != null) { - try { - result.cursorData = JSON.parse(result.cursorData) - } catch (e) { - logger.error( - { - err: e, - project_id, - client_id, - cursorData: result.cursorData - }, - 'error parsing cursorData JSON' - ) - return callback(e) - } + rclient.hgetall(Keys.connectedUser({ project_id, client_id }), function ( + err, + result + ) { + if (!(result && result.user_id)) { + result = { + connected: false, + client_id + } + } else { + result.connected = true + result.client_id = client_id + result.client_age = + (Date.now() - parseInt(result.last_updated_at, 10)) / 1000 + if (result.cursorData) { + try { + result.cursorData = JSON.parse(result.cursorData) + } catch (e) { + logger.error( + { + err: e, + project_id, + client_id, + cursorData: result.cursorData + }, + 'error parsing cursorData JSON' + ) + return callback(e) } } - return callback(err, result) } - ) + callback(err, result) + }) }, getConnectedUsers(project_id, callback) { const self = this - return rclient.smembers(Keys.clientsInProject({ project_id }), function ( + rclient.smembers(Keys.clientsInProject({ project_id }), function ( err, results ) { - if (err != null) { + if (err) { return callback(err) } const jobs = results.map((client_id) => (cb) => self._getConnectedUser(project_id, client_id, cb) ) - return async.series(jobs, function (err, users) { - if (users == null) { - users = [] - } - if (err != null) { + async.series(jobs, function (err, users) { + if (err) { return callback(err) } users = users.filter( (user) => - (user != null ? user.connected : undefined) && - (user != null ? user.client_age : undefined) < REFRESH_TIMEOUT_IN_S + user && user.connected && user.client_age < REFRESH_TIMEOUT_IN_S ) - return callback(null, users) + callback(null, users) }) }) } diff --git a/services/real-time/app/js/DocumentUpdaterController.js b/services/real-time/app/js/DocumentUpdaterController.js index b8dde3b426..bf8d25ae85 100644 --- a/services/real-time/app/js/DocumentUpdaterController.js +++ b/services/real-time/app/js/DocumentUpdaterController.js @@ -1,18 +1,6 @@ /* eslint-disable camelcase, - no-unused-vars, */ -// TODO: This file was created by bulk-decaffeinate. -// Fix any style issues and re-enable lint. -/* - * decaffeinate suggestions: - * DS101: Remove unnecessary use of Array.from - * DS102: Remove unnecessary code created because of implicit returns - * DS205: Consider reworking code to avoid use of IIFEs - * DS207: Consider shorter variations of null checks - * Full docs: https://github.com/decaffeinate/decaffeinate/blob/master/docs/suggestions.md - */ -let DocumentUpdaterController const logger = require('logger-sharelatex') const settings = require('settings-sharelatex') const RedisClientManager = require('./RedisClientManager') @@ -23,28 +11,25 @@ const RoomManager = require('./RoomManager') const ChannelManager = require('./ChannelManager') const metrics = require('metrics-sharelatex') -const MESSAGE_SIZE_LOG_LIMIT = 1024 * 1024 // 1Mb - +let DocumentUpdaterController module.exports = DocumentUpdaterController = { // DocumentUpdaterController is responsible for updates that come via Redis // Pub/Sub from the document updater. rclientList: RedisClientManager.createClientList(settings.redis.pubsub), listenForUpdatesFromDocumentUpdater(io) { - let i, rclient logger.log( { rclients: this.rclientList.length }, 'listening for applied-ops events' ) - for (i = 0; i < this.rclientList.length; i++) { - rclient = this.rclientList[i] + for (const rclient of this.rclientList) { rclient.subscribe('applied-ops') rclient.on('message', function (channel, message) { metrics.inc('rclient', 0.001) // global event rate metric if (settings.debugEvents > 0) { EventLogger.debugEvent(channel, message) } - return DocumentUpdaterController._processMessageFromDocumentUpdater( + DocumentUpdaterController._processMessageFromDocumentUpdater( io, channel, message @@ -53,42 +38,41 @@ module.exports = DocumentUpdaterController = { } // create metrics for each redis instance only when we have multiple redis clients if (this.rclientList.length > 1) { - for (i = 0; i < this.rclientList.length; i++) { - rclient = this.rclientList[i] - ;(( - i // per client event rate metric - ) => rclient.on('message', () => metrics.inc(`rclient-${i}`, 0.001)))(i) - } + this.rclientList.forEach((rclient, i) => { + // per client event rate metric + const metricName = `rclient-${i}` + rclient.on('message', () => metrics.inc(metricName, 0.001)) + }) } - return this.handleRoomUpdates(this.rclientList) + this.handleRoomUpdates(this.rclientList) }, handleRoomUpdates(rclientSubList) { const roomEvents = RoomManager.eventSource() roomEvents.on('doc-active', function (doc_id) { - const subscribePromises = Array.from(rclientSubList).map((rclient) => + const subscribePromises = rclientSubList.map((rclient) => ChannelManager.subscribe(rclient, 'applied-ops', doc_id) ) - return RoomManager.emitOnCompletion( + RoomManager.emitOnCompletion( subscribePromises, `doc-subscribed-${doc_id}` ) }) - return roomEvents.on('doc-empty', (doc_id) => - Array.from(rclientSubList).map((rclient) => + roomEvents.on('doc-empty', (doc_id) => + rclientSubList.map((rclient) => ChannelManager.unsubscribe(rclient, 'applied-ops', doc_id) ) ) }, _processMessageFromDocumentUpdater(io, channel, message) { - return SafeJsonParse.parse(message, function (error, message) { - if (error != null) { + SafeJsonParse.parse(message, function (error, message) { + if (error) { logger.error({ err: error, channel }, 'error parsing JSON') return } - if (message.op != null) { - if (message._id != null && settings.checkEventOrder) { + if (message.op) { + if (message._id && settings.checkEventOrder) { const status = EventLogger.checkEventOrder( 'applied-ops', message._id, @@ -98,24 +82,24 @@ module.exports = DocumentUpdaterController = { return // skip duplicate events } } - return DocumentUpdaterController._applyUpdateFromDocumentUpdater( + DocumentUpdaterController._applyUpdateFromDocumentUpdater( io, message.doc_id, message.op ) - } else if (message.error != null) { - return DocumentUpdaterController._processErrorFromDocumentUpdater( + } else if (message.error) { + DocumentUpdaterController._processErrorFromDocumentUpdater( io, message.doc_id, message.error, message ) - } else if (message.health_check != null) { + } else if (message.health_check) { logger.debug( { message }, 'got health check message in applied ops channel' ) - return HealthCheckManager.check(channel, message.key) + HealthCheckManager.check(channel, message.key) } }) }, @@ -132,20 +116,14 @@ module.exports = DocumentUpdaterController = { { doc_id, version: update.v, - source: update.meta != null ? update.meta.source : undefined, - socketIoClients: (() => { - const result = [] - for (client of Array.from(clientList)) { - result.push(client.id) - } - return result - })() + source: update.meta && update.meta.source, + socketIoClients: clientList.map((client) => client.id) }, 'distributing updates to clients' ) const seen = {} // send messages only to unique clients (due to duplicate entries in io.sockets.clients) - for (client of Array.from(clientList)) { + for (client of clientList) { if (!seen[client.id]) { seen[client.id] = true if (client.publicId === update.meta.source) { @@ -153,7 +131,7 @@ module.exports = DocumentUpdaterController = { { doc_id, version: update.v, - source: update.meta != null ? update.meta.source : undefined + source: update.meta.source }, 'distributing update to sender' ) @@ -164,7 +142,7 @@ module.exports = DocumentUpdaterController = { { doc_id, version: update.v, - source: update.meta != null ? update.meta.source : undefined, + source: update.meta.source, client_id: client.id }, 'distributing update to collaborator' @@ -175,16 +153,10 @@ module.exports = DocumentUpdaterController = { } if (Object.keys(seen).length < clientList.length) { metrics.inc('socket-io.duplicate-clients', 0.1) - return logger.log( + logger.log( { doc_id, - socketIoClients: (() => { - const result1 = [] - for (client of Array.from(clientList)) { - result1.push(client.id) - } - return result1 - })() + socketIoClients: clientList.map((client) => client.id) }, 'discarded duplicate clients' ) @@ -192,17 +164,13 @@ module.exports = DocumentUpdaterController = { }, _processErrorFromDocumentUpdater(io, doc_id, error, message) { - return (() => { - const result = [] - for (const client of Array.from(io.sockets.clients(doc_id))) { - logger.warn( - { err: error, doc_id, client_id: client.id }, - 'error from document updater, disconnecting client' - ) - client.emit('otUpdateError', error, message) - result.push(client.disconnect()) - } - return result - })() + for (const client of io.sockets.clients(doc_id)) { + logger.warn( + { err: error, doc_id, client_id: client.id }, + 'error from document updater, disconnecting client' + ) + client.emit('otUpdateError', error, message) + client.disconnect() + } } } diff --git a/services/real-time/app/js/DocumentUpdaterManager.js b/services/real-time/app/js/DocumentUpdaterManager.js index bafc81ed14..32533a9eaa 100644 --- a/services/real-time/app/js/DocumentUpdaterManager.js +++ b/services/real-time/app/js/DocumentUpdaterManager.js @@ -1,17 +1,6 @@ /* eslint-disable camelcase, - handle-callback-err, - no-unused-vars, */ -// TODO: This file was created by bulk-decaffeinate. -// Fix any style issues and re-enable lint. -/* - * decaffeinate suggestions: - * DS102: Remove unnecessary code created because of implicit returns - * DS207: Consider shorter variations of null checks - * Full docs: https://github.com/decaffeinate/decaffeinate/blob/master/docs/suggestions.md - */ -let DocumentUpdaterManager const request = require('request') const _ = require('underscore') const logger = require('logger-sharelatex') @@ -23,20 +12,17 @@ const rclient = require('redis-sharelatex').createClient( ) const Keys = settings.redis.documentupdater.key_schema -module.exports = DocumentUpdaterManager = { +module.exports = { getDocument(project_id, doc_id, fromVersion, callback) { - if (callback == null) { - callback = function (error, exists, doclines, version) {} - } const timer = new metrics.Timer('get-document') const url = `${settings.apis.documentupdater.url}/project/${project_id}/doc/${doc_id}?fromVersion=${fromVersion}` logger.log( { project_id, doc_id, fromVersion }, 'getting doc from document updater' ) - return request.get(url, function (err, res, body) { + request.get(url, function (err, res, body) { timer.done() - if (err != null) { + if (err) { logger.error( { err, url, project_id, doc_id }, 'error getting doc from doc updater' @@ -53,13 +39,8 @@ module.exports = DocumentUpdaterManager = { } catch (error) { return callback(error) } - return callback( - null, - body != null ? body.lines : undefined, - body != null ? body.version : undefined, - body != null ? body.ranges : undefined, - body != null ? body.ops : undefined - ) + body = body || {} + callback(null, body.lines, body.version, body.ranges, body.ops) } else if ([404, 422].includes(res.statusCode)) { err = new Error('doc updater could not load requested ops') err.statusCode = res.statusCode @@ -67,7 +48,7 @@ module.exports = DocumentUpdaterManager = { { err, project_id, doc_id, url, fromVersion }, 'doc updater could not load requested ops' ) - return callback(err) + callback(err) } else { err = new Error( `doc updater returned a non-success status code: ${res.statusCode}` @@ -77,33 +58,30 @@ module.exports = DocumentUpdaterManager = { { err, project_id, doc_id, url }, `doc updater returned a non-success status code: ${res.statusCode}` ) - return callback(err) + callback(err) } }) }, flushProjectToMongoAndDelete(project_id, callback) { // this method is called when the last connected user leaves the project - if (callback == null) { - callback = function () {} - } logger.log({ project_id }, 'deleting project from document updater') const timer = new metrics.Timer('delete.mongo.project') // flush the project in the background when all users have left const url = `${settings.apis.documentupdater.url}/project/${project_id}?background=true` + (settings.shutDownInProgress ? '&shutdown=true' : '') - return request.del(url, function (err, res, body) { + request.del(url, function (err, res) { timer.done() - if (err != null) { + if (err) { logger.error( { err, project_id }, 'error deleting project from document updater' ) - return callback(err) + callback(err) } else if (res.statusCode >= 200 && res.statusCode < 300) { logger.log({ project_id }, 'deleted project from document updater') - return callback(null) + callback(null) } else { err = new Error( `document updater returned a failure status code: ${res.statusCode}` @@ -113,16 +91,12 @@ module.exports = DocumentUpdaterManager = { { err, project_id }, `document updater returned failure status code: ${res.statusCode}` ) - return callback(err) + callback(err) } }) }, queueChange(project_id, doc_id, change, callback) { - let error - if (callback == null) { - callback = function () {} - } const allowedKeys = [ 'doc', 'op', @@ -136,7 +110,7 @@ module.exports = DocumentUpdaterManager = { const jsonChange = JSON.stringify(change) if (jsonChange.indexOf('\u0000') !== -1) { // memory corruption check - error = new Error('null bytes found in op') + const error = new Error('null bytes found in op') logger.error( { err: error, project_id, doc_id, jsonChange }, error.message @@ -146,7 +120,7 @@ module.exports = DocumentUpdaterManager = { const updateSize = jsonChange.length if (updateSize > settings.maxUpdateSize) { - error = new Error('update is too large') + const error = new Error('update is too large') error.updateSize = updateSize return callback(error) } @@ -157,13 +131,13 @@ module.exports = DocumentUpdaterManager = { const doc_key = `${project_id}:${doc_id}` // Push onto pendingUpdates for doc_id first, because once the doc updater // gets an entry on pending-updates-list, it starts processing. - return rclient.rpush(Keys.pendingUpdates({ doc_id }), jsonChange, function ( + rclient.rpush(Keys.pendingUpdates({ doc_id }), jsonChange, function ( error ) { - if (error != null) { + if (error) { return callback(error) } - return rclient.rpush('pending-updates-list', doc_key, callback) + rclient.rpush('pending-updates-list', doc_key, callback) }) } } diff --git a/services/real-time/app/js/DrainManager.js b/services/real-time/app/js/DrainManager.js index b8c08356bb..c605957160 100644 --- a/services/real-time/app/js/DrainManager.js +++ b/services/real-time/app/js/DrainManager.js @@ -1,31 +1,21 @@ -/* eslint-disable - no-return-assign, -*/ -// TODO: This file was created by bulk-decaffeinate. -// Fix any style issues and re-enable lint. -/* - * decaffeinate suggestions: - * DS101: Remove unnecessary use of Array.from - * DS102: Remove unnecessary code created because of implicit returns - * Full docs: https://github.com/decaffeinate/decaffeinate/blob/master/docs/suggestions.md - */ -let DrainManager const logger = require('logger-sharelatex') -module.exports = DrainManager = { +module.exports = { startDrainTimeWindow(io, minsToDrain) { const drainPerMin = io.sockets.clients().length / minsToDrain - return DrainManager.startDrain(io, Math.max(drainPerMin / 60, 4)) - }, // enforce minimum drain rate + // enforce minimum drain rate + this.startDrain(io, Math.max(drainPerMin / 60, 4)) + }, startDrain(io, rate) { // Clear out any old interval - let pollingInterval clearInterval(this.interval) logger.log({ rate }, 'starting drain') if (rate === 0) { return - } else if (rate < 1) { + } + let pollingInterval + if (rate < 1) { // allow lower drain rates // e.g. rate=0.1 will drain one client every 10 seconds pollingInterval = 1000 / rate @@ -33,15 +23,15 @@ module.exports = DrainManager = { } else { pollingInterval = 1000 } - return (this.interval = setInterval(() => { - return this.reconnectNClients(io, rate) - }, pollingInterval)) + this.interval = setInterval(() => { + this.reconnectNClients(io, rate) + }, pollingInterval) }, RECONNECTED_CLIENTS: {}, reconnectNClients(io, N) { let drainedCount = 0 - for (const client of Array.from(io.sockets.clients())) { + for (const client of io.sockets.clients()) { if (!this.RECONNECTED_CLIENTS[client.id]) { this.RECONNECTED_CLIENTS[client.id] = true logger.log( @@ -57,7 +47,7 @@ module.exports = DrainManager = { } } if (drainedCount < N) { - return logger.log('All clients have been told to reconnectGracefully') + logger.log('All clients have been told to reconnectGracefully') } } } diff --git a/services/real-time/app/js/Errors.js b/services/real-time/app/js/Errors.js index 8bfe3763b0..bdda1b4c21 100644 --- a/services/real-time/app/js/Errors.js +++ b/services/real-time/app/js/Errors.js @@ -1,17 +1,9 @@ -/* eslint-disable - no-proto, - no-unused-vars, -*/ -// TODO: This file was created by bulk-decaffeinate. -// Fix any style issues and re-enable lint. -let Errors -var CodedError = function (message, code) { - const error = new Error(message) - error.name = 'CodedError' - error.code = code - error.__proto__ = CodedError.prototype - return error +class CodedError extends Error { + constructor(message, code) { + super(message) + this.name = this.constructor.name + this.code = code + } } -CodedError.prototype.__proto__ = Error.prototype -module.exports = Errors = { CodedError } +module.exports = { CodedError } diff --git a/services/real-time/app/js/EventLogger.js b/services/real-time/app/js/EventLogger.js index 1133ebdaf8..9db3f8aa55 100644 --- a/services/real-time/app/js/EventLogger.js +++ b/services/real-time/app/js/EventLogger.js @@ -1,15 +1,6 @@ /* eslint-disable camelcase, */ -// TODO: This file was created by bulk-decaffeinate. -// Fix any style issues and re-enable lint. -/* - * decaffeinate suggestions: - * DS102: Remove unnecessary code created because of implicit returns - * DS205: Consider reworking code to avoid use of IIFEs - * DS207: Consider shorter variations of null checks - * Full docs: https://github.com/decaffeinate/decaffeinate/blob/master/docs/suggestions.md - */ let EventLogger const logger = require('logger-sharelatex') const metrics = require('metrics-sharelatex') @@ -31,15 +22,15 @@ module.exports = EventLogger = { debugEvent(channel, message) { if (settings.debugEvents > 0) { logger.log({ channel, message, counter: COUNTER++ }, 'logging event') - return settings.debugEvents-- + settings.debugEvents-- } }, - checkEventOrder(channel, message_id, message) { - let result + checkEventOrder(channel, message_id) { if (typeof message_id !== 'string') { return } + let result if (!(result = message_id.match(/^(.*)-(\d+)$/))) { return } @@ -51,7 +42,7 @@ module.exports = EventLogger = { } // store the last count in a hash for each host const previous = EventLogger._storeEventCount(key, count) - if (previous == null || count === previous + 1) { + if (!previous || count === previous + 1) { metrics.inc(`event.${channel}.valid`, 0.001) // downsample high rate docupdater events return // order is ok } @@ -83,18 +74,11 @@ module.exports = EventLogger = { }, _cleanEventStream(now) { - return (() => { - const result = [] - for (const key in EVENT_LOG_TIMESTAMP) { - const timestamp = EVENT_LOG_TIMESTAMP[key] - if (now - timestamp > EventLogger.MAX_STALE_TIME_IN_MS) { - delete EVENT_LOG_COUNTER[key] - result.push(delete EVENT_LOG_TIMESTAMP[key]) - } else { - result.push(undefined) - } + Object.entries(EVENT_LOG_TIMESTAMP).forEach(([key, timestamp]) => { + if (now - timestamp > EventLogger.MAX_STALE_TIME_IN_MS) { + delete EVENT_LOG_COUNTER[key] + delete EVENT_LOG_TIMESTAMP[key] } - return result - })() + }) } } diff --git a/services/real-time/app/js/HealthCheckManager.js b/services/real-time/app/js/HealthCheckManager.js index 4704aa5e88..a297807fe8 100644 --- a/services/real-time/app/js/HealthCheckManager.js +++ b/services/real-time/app/js/HealthCheckManager.js @@ -1,16 +1,3 @@ -/* eslint-disable - no-return-assign, - no-unused-vars, -*/ -// TODO: This file was created by bulk-decaffeinate. -// Fix any style issues and re-enable lint. -/* - * decaffeinate suggestions: - * DS102: Remove unnecessary code created because of implicit returns - * DS207: Consider shorter variations of null checks - * Full docs: https://github.com/decaffeinate/decaffeinate/blob/master/docs/suggestions.md - */ -let HealthCheckManager const metrics = require('metrics-sharelatex') const logger = require('logger-sharelatex') @@ -22,22 +9,19 @@ let COUNT = 0 const CHANNEL_MANAGER = {} // hash of event checkers by channel name const CHANNEL_ERROR = {} // error status by channel name -module.exports = HealthCheckManager = class HealthCheckManager { +module.exports = class HealthCheckManager { // create an instance of this class which checks that an event with a unique // id is received only once within a timeout constructor(channel, timeout) { // unique event string this.channel = channel - if (timeout == null) { - timeout = 1000 - } this.id = `host=${HOST}:pid=${PID}:count=${COUNT++}` // count of number of times the event is received this.count = 0 // after a timeout check the status of the count this.handler = setTimeout(() => { - return this.setStatus() - }, timeout) + this.setStatus() + }, timeout || 1000) // use a timer to record the latency of the channel this.timer = new metrics.Timer(`event.${this.channel}.latency`) // keep a record of these objects to dispatch on @@ -48,31 +32,31 @@ module.exports = HealthCheckManager = class HealthCheckManager { // if this is our event record it if (id === this.id) { this.count++ - if (this.timer != null) { + if (this.timer) { this.timer.done() } - return (this.timer = null) // only time the latency of the first event + this.timer = undefined // only time the latency of the first event } } setStatus() { // if we saw the event anything other than a single time that is an error - if (this.count !== 1) { + const isFailing = this.count !== 1 + if (isFailing) { logger.err( { channel: this.channel, count: this.count, id: this.id }, 'redis channel health check error' ) } - const error = this.count !== 1 - return (CHANNEL_ERROR[this.channel] = error) + CHANNEL_ERROR[this.channel] = isFailing } // class methods static check(channel, id) { // dispatch event to manager for channel - return CHANNEL_MANAGER[channel] != null - ? CHANNEL_MANAGER[channel].processEvent(id) - : undefined + if (CHANNEL_MANAGER[channel]) { + CHANNEL_MANAGER[channel].processEvent(id) + } } static status() { diff --git a/services/real-time/app/js/HttpApiController.js b/services/real-time/app/js/HttpApiController.js index a512961797..7b9627fd7b 100644 --- a/services/real-time/app/js/HttpApiController.js +++ b/services/real-time/app/js/HttpApiController.js @@ -1,25 +1,15 @@ /* eslint-disable camelcase, - no-unused-vars, */ -// TODO: This file was created by bulk-decaffeinate. -// Fix any style issues and re-enable lint. -/* - * decaffeinate suggestions: - * DS101: Remove unnecessary use of Array.from - * DS102: Remove unnecessary code created because of implicit returns - * Full docs: https://github.com/decaffeinate/decaffeinate/blob/master/docs/suggestions.md - */ -let HttpApiController const WebsocketLoadBalancer = require('./WebsocketLoadBalancer') const DrainManager = require('./DrainManager') const logger = require('logger-sharelatex') -module.exports = HttpApiController = { - sendMessage(req, res, next) { +module.exports = { + sendMessage(req, res) { logger.log({ message: req.params.message }, 'sending message') if (Array.isArray(req.body)) { - for (const payload of Array.from(req.body)) { + for (const payload of req.body) { WebsocketLoadBalancer.emitToRoom( req.params.project_id, req.params.message, @@ -33,16 +23,16 @@ module.exports = HttpApiController = { req.body ) } - return res.send(204) - }, // No content + res.send(204) + }, - startDrain(req, res, next) { + startDrain(req, res) { const io = req.app.get('io') let rate = req.query.rate || '4' rate = parseFloat(rate) || 0 logger.log({ rate }, 'setting client drain rate') DrainManager.startDrain(io, rate) - return res.send(204) + res.send(204) }, disconnectClient(req, res, next) { @@ -57,6 +47,6 @@ module.exports = HttpApiController = { } logger.warn({ client_id }, 'api: requesting client disconnect') client.on('disconnect', () => res.sendStatus(204)) - return client.disconnect() + client.disconnect() } } diff --git a/services/real-time/app/js/HttpController.js b/services/real-time/app/js/HttpController.js index deabf5876d..68e5d4cd07 100644 --- a/services/real-time/app/js/HttpController.js +++ b/services/real-time/app/js/HttpController.js @@ -1,28 +1,15 @@ /* eslint-disable camelcase, - handle-callback-err, */ -// TODO: This file was created by bulk-decaffeinate. -// Fix any style issues and re-enable lint. -/* - * decaffeinate suggestions: - * DS102: Remove unnecessary code created because of implicit returns - * DS207: Consider shorter variations of null checks - * Full docs: https://github.com/decaffeinate/decaffeinate/blob/master/docs/suggestions.md - */ -let HttpController -const async = require('async') +let HttpController module.exports = HttpController = { // The code in this controller is hard to unit test because of a lot of // dependencies on internal socket.io methods. It is not critical to the running // of ShareLaTeX, and is only used for getting stats about connected clients, // and for checking internal state in acceptance tests. The acceptances tests // should provide appropriate coverage. - _getConnectedClientView(ioClient, callback) { - if (callback == null) { - callback = function (error, client) {} - } + _getConnectedClientView(ioClient) { const client_id = ioClient.id const { project_id, @@ -41,32 +28,23 @@ module.exports = HttpController = { email, connected_time } - client.rooms = [] - for (const name in ioClient.manager.roomClients[client_id]) { - const joined = ioClient.manager.roomClients[client_id][name] - if (joined && name !== '') { - client.rooms.push(name.replace(/^\//, '')) // Remove leading / - } - } - return callback(null, client) + client.rooms = Object.keys(ioClient.manager.roomClients[client_id] || {}) + // drop the namespace + .filter((room) => room !== '') + // room names are composed as '/' and the default + // namespace is empty (see comments in RoomManager), just drop the '/' + .map((fullRoomPath) => fullRoomPath.slice(1)) + return client }, - getConnectedClients(req, res, next) { + getConnectedClients(req, res) { const io = req.app.get('io') const ioClients = io.sockets.clients() - return async.map( - ioClients, - HttpController._getConnectedClientView, - function (error, clients) { - if (error != null) { - return next(error) - } - return res.json(clients) - } - ) + + res.json(ioClients.map(HttpController._getConnectedClientView)) }, - getConnectedClient(req, res, next) { + getConnectedClient(req, res) { const { client_id } = req.params const io = req.app.get('io') const ioClient = io.sockets.sockets[client_id] @@ -74,14 +52,6 @@ module.exports = HttpController = { res.sendStatus(404) return } - return HttpController._getConnectedClientView(ioClient, function ( - error, - client - ) { - if (error != null) { - return next(error) - } - return res.json(client) - }) + res.json(HttpController._getConnectedClientView(ioClient)) } } diff --git a/services/real-time/app/js/RedisClientManager.js b/services/real-time/app/js/RedisClientManager.js index b43262aeda..d05cc64818 100644 --- a/services/real-time/app/js/RedisClientManager.js +++ b/services/real-time/app/js/RedisClientManager.js @@ -1,40 +1,19 @@ -/* eslint-disable - no-unused-vars, -*/ -// TODO: This file was created by bulk-decaffeinate. -// Fix any style issues and re-enable lint. -/* - * decaffeinate suggestions: - * DS101: Remove unnecessary use of Array.from - * DS205: Consider reworking code to avoid use of IIFEs - * DS207: Consider shorter variations of null checks - * Full docs: https://github.com/decaffeinate/decaffeinate/blob/master/docs/suggestions.md - */ -let RedisClientManager const redis = require('redis-sharelatex') const logger = require('logger-sharelatex') -module.exports = RedisClientManager = { +module.exports = { createClientList(...configs) { // create a dynamic list of redis clients, excluding any configurations which are not defined - const clientList = (() => { - const result = [] - for (const x of Array.from(configs)) { - if (x != null) { - const redisType = - x.cluster != null - ? 'cluster' - : x.sentinels != null - ? 'sentinel' - : x.host != null - ? 'single' - : 'unknown' - logger.log({ redis: redisType }, 'creating redis client') - result.push(redis.createClient(x)) - } - } - return result - })() - return clientList + return configs.filter(Boolean).map((x) => { + const redisType = x.cluster + ? 'cluster' + : x.sentinels + ? 'sentinel' + : x.host + ? 'single' + : 'unknown' + logger.log({ redis: redisType }, 'creating redis client') + return redis.createClient(x) + }) } } diff --git a/services/real-time/app/js/RoomManager.js b/services/real-time/app/js/RoomManager.js index 8dd34e9340..81827e5e9a 100644 --- a/services/real-time/app/js/RoomManager.js +++ b/services/real-time/app/js/RoomManager.js @@ -1,19 +1,6 @@ /* eslint-disable camelcase, - no-unused-vars, */ -// TODO: This file was created by bulk-decaffeinate. -// Fix any style issues and re-enable lint. -/* - * decaffeinate suggestions: - * DS101: Remove unnecessary use of Array.from - * DS102: Remove unnecessary code created because of implicit returns - * DS103: Rewrite code to no longer use __guard__ - * DS205: Consider reworking code to avoid use of IIFEs - * DS207: Consider shorter variations of null checks - * Full docs: https://github.com/decaffeinate/decaffeinate/blob/master/docs/suggestions.md - */ -let RoomManager const logger = require('logger-sharelatex') const metrics = require('metrics-sharelatex') const { EventEmitter } = require('events') @@ -31,23 +18,17 @@ const RoomEvents = new EventEmitter() // emits {project,doc}-active and {project // // The pubsub side is handled by ChannelManager -module.exports = RoomManager = { +module.exports = { joinProject(client, project_id, callback) { - if (callback == null) { - callback = function () {} - } - return this.joinEntity(client, 'project', project_id, callback) + this.joinEntity(client, 'project', project_id, callback) }, joinDoc(client, doc_id, callback) { - if (callback == null) { - callback = function () {} - } - return this.joinEntity(client, 'doc', doc_id, callback) + this.joinEntity(client, 'doc', doc_id, callback) }, leaveDoc(client, doc_id) { - return this.leaveEntity(client, 'doc', doc_id) + this.leaveEntity(client, 'doc', doc_id) }, leaveProjectAndDocs(client) { @@ -58,18 +39,14 @@ module.exports = RoomManager = { // has not joined any rooms and do a final disconnection. const roomsToLeave = this._roomsClientIsIn(client) logger.log({ client: client.id, roomsToLeave }, 'client leaving project') - return (() => { - const result = [] - for (const id of Array.from(roomsToLeave)) { - const entity = IdMap.get(id) - result.push(this.leaveEntity(client, entity, id)) - } - return result - })() + for (const id of roomsToLeave) { + const entity = IdMap.get(id) + this.leaveEntity(client, entity, id) + } }, emitOnCompletion(promiseList, eventName) { - return Promise.all(promiseList) + Promise.all(promiseList) .then(() => RoomEvents.emit(eventName)) .catch((err) => RoomEvents.emit(eventName, err)) }, @@ -92,19 +69,19 @@ module.exports = RoomManager = { { client: client.id, entity, id, beforeCount }, 'client joined new room and subscribed to channel' ) - return callback(err) + callback(err) }) RoomEvents.emit(`${entity}-active`, id) IdMap.set(id, entity) // keep track of the number of listeners - return metrics.gauge('room-listeners', RoomEvents.eventNames().length) + metrics.gauge('room-listeners', RoomEvents.eventNames().length) } else { logger.log( { client: client.id, entity, id, beforeCount }, 'client joined existing room' ) client.join(id) - return callback() + callback() } }, @@ -128,7 +105,7 @@ module.exports = RoomManager = { 'client left room' ) // is the room now empty? if so, unsubscribe - if (entity == null) { + if (!entity) { logger.error({ entity: id }, 'unknown entity when leaving with id') return } @@ -136,54 +113,48 @@ module.exports = RoomManager = { logger.log({ entity, id }, 'room is now empty') RoomEvents.emit(`${entity}-empty`, id) IdMap.delete(id) - return metrics.gauge('room-listeners', RoomEvents.eventNames().length) + metrics.gauge('room-listeners', RoomEvents.eventNames().length) } }, // internal functions below, these access socket.io rooms data directly and // will need updating for socket.io v2 + // The below code makes some assumptions that are always true for v0 + // - we are using the base namespace '', so room names are '/' + // https://github.com/socketio/socket.io/blob/e4d61b1be65ac3313a85da111a46777aa8d4aae3/lib/manager.js#L62 + // https://github.com/socketio/socket.io/blob/e4d61b1be65ac3313a85da111a46777aa8d4aae3/lib/manager.js#L1018 + // - client.namespace is a Namespace + // https://github.com/socketio/socket.io/blob/e4d61b1be65ac3313a85da111a46777aa8d4aae3/lib/namespace.js#L204 + // https://github.com/socketio/socket.io/blob/e4d61b1be65ac3313a85da111a46777aa8d4aae3/lib/socket.js#L40 + // - client.manager is a Manager + // https://github.com/socketio/socket.io/blob/e4d61b1be65ac3313a85da111a46777aa8d4aae3/lib/namespace.js#L204 + // https://github.com/socketio/socket.io/blob/e4d61b1be65ac3313a85da111a46777aa8d4aae3/lib/socket.js#L41 + // - a Manager has + // - `.rooms={'NAMESPACE/ENTITY': []}` and + // - `.roomClients={'CLIENT_ID': {'...': true}}` + // https://github.com/socketio/socket.io/blob/e4d61b1be65ac3313a85da111a46777aa8d4aae3/lib/manager.js#L287-L288 + // https://github.com/socketio/socket.io/blob/e4d61b1be65ac3313a85da111a46777aa8d4aae3/lib/manager.js#L444-L455 + _clientsInRoom(client, room) { - const nsp = client.namespace.name - const name = nsp + '/' + room - return ( - __guard__( - client.manager != null ? client.manager.rooms : undefined, - (x) => x[name] - ) || [] - ).length + const clients = client.manager.rooms['/' + room] || [] + return clients.length }, _roomsClientIsIn(client) { - const roomList = (() => { - const result = [] - for (const fullRoomPath in client.manager.roomClients != null - ? client.manager.roomClients[client.id] - : undefined) { - // strip socket.io prefix from room to get original id - if (fullRoomPath !== '') { - const [prefix, room] = Array.from(fullRoomPath.split('/', 2)) - result.push(room) - } - } - return result - })() - return roomList + const rooms = client.manager.roomClients[client.id] || {} + return ( + Object.keys(rooms) + // drop the namespace + .filter((room) => room !== '') + // room names are composed as '/' and the default + // namespace is empty (see comments above), just drop the '/' + .map((fullRoomPath) => fullRoomPath.slice(1)) + ) }, _clientAlreadyInRoom(client, room) { - const nsp = client.namespace.name - const name = nsp + '/' + room - return __guard__( - client.manager.roomClients != null - ? client.manager.roomClients[client.id] - : undefined, - (x) => x[name] - ) + const rooms = client.manager.roomClients[client.id] || {} + return !!rooms['/' + room] } } -function __guard__(value, transform) { - return typeof value !== 'undefined' && value !== null - ? transform(value) - : undefined -} diff --git a/services/real-time/app/js/Router.js b/services/real-time/app/js/Router.js index 0e19c46bc0..02c751c22f 100644 --- a/services/real-time/app/js/Router.js +++ b/services/real-time/app/js/Router.js @@ -1,19 +1,6 @@ /* eslint-disable camelcase, - handle-callback-err, - standard/no-callback-literal, */ -// TODO: This file was created by bulk-decaffeinate. -// Fix any style issues and re-enable lint. -/* - * decaffeinate suggestions: - * DS101: Remove unnecessary use of Array.from - * DS102: Remove unnecessary code created because of implicit returns - * DS103: Rewrite code to no longer use __guard__ - * DS207: Consider shorter variations of null checks - * Full docs: https://github.com/decaffeinate/decaffeinate/blob/master/docs/suggestions.md - */ -let Router const metrics = require('metrics-sharelatex') const logger = require('logger-sharelatex') const settings = require('settings-sharelatex') @@ -34,14 +21,10 @@ const httpAuth = basicAuth(function (user, pass) { return isValid }) +let Router module.exports = Router = { _handleError(callback, error, client, method, attrs) { - if (callback == null) { - callback = function (error) {} - } - if (attrs == null) { - attrs = {} - } + attrs = attrs || {} for (const key of ['project_id', 'doc_id', 'user_id']) { attrs[key] = client.ol_context[key] } @@ -49,15 +32,15 @@ module.exports = Router = { attrs.err = error if (error.name === 'CodedError') { logger.warn(attrs, error.message, { code: error.code }) - return callback({ message: error.message, code: error.code }) - } - if (error.message === 'unexpected arguments') { + const serializedError = { message: error.message, code: error.code } + callback(serializedError) + } else if (error.message === 'unexpected arguments') { // the payload might be very large, put it on level info logger.log(attrs, 'unexpected arguments') metrics.inc('unexpected-arguments', 1, { status: method }) - return callback({ message: error.message }) - } - if ( + const serializedError = { message: error.message } + callback(serializedError) + } else if ( [ 'not authorized', 'doc updater could not load requested ops', @@ -65,11 +48,15 @@ module.exports = Router = { ].includes(error.message) ) { logger.warn(attrs, error.message) - return callback({ message: error.message }) + const serializedError = { message: error.message } + callback(serializedError) } else { logger.error(attrs, `server side error in ${method}`) // Don't return raw error to prevent leaking server side info - return callback({ message: 'Something went wrong in real-time service' }) + const serializedError = { + message: 'Something went wrong in real-time service' + } + callback(serializedError) } }, @@ -80,7 +67,7 @@ module.exports = Router = { callback = function () {} } const attrs = { arguments: args } - return Router._handleError(callback, error, client, method, attrs) + Router._handleError(callback, error, client, method, attrs) }, configure(app, io, session) { @@ -102,18 +89,17 @@ module.exports = Router = { HttpApiController.disconnectClient ) - return session.on('connection', function (error, client, session) { + session.on('connection', function (error, client, session) { // init client context, we may access it in Router._handleError before // setting any values - let user client.ol_context = {} - if (client != null) { + if (client) { client.on('error', function (err) { logger.err({ clientErr: err }, 'socket.io client error') if (client.connected) { client.emit('reconnectGracefully') - return client.disconnect() + client.disconnect() } }) } @@ -125,13 +111,12 @@ module.exports = Router = { } if ( - client != null && - __guard__(error != null ? error.message : undefined, (x) => - x.match(/could not look up session by key/) - ) + client && + error && + error.message.match(/could not look up session by key/) ) { logger.warn( - { err: error, client: client != null, session: session != null }, + { err: error, client: !!client, session: !!session }, 'invalid session' ) // tell the client to reauthenticate if it has an invalid session key @@ -140,15 +125,15 @@ module.exports = Router = { return } - if (error != null) { + if (error) { logger.err( - { err: error, client: client != null, session: session != null }, + { err: error, client: !!client, session: !!session }, 'error when client connected' ) - if (client != null) { + if (client) { client.emit('connectionRejected', { message: 'error' }) } - if (client != null) { + if (client) { client.disconnect() } return @@ -159,30 +144,21 @@ module.exports = Router = { client.emit('connectionAccepted', null, client.publicId) metrics.inc('socket-io.connection') - metrics.gauge( - 'socket-io.clients', - __guard__(io.sockets.clients(), (x1) => x1.length) - ) + metrics.gauge('socket-io.clients', io.sockets.clients().length) logger.log({ session, client_id: client.id }, 'client connected') - if ( - __guard__( - session != null ? session.passport : undefined, - (x2) => x2.user - ) != null - ) { + let user + if (session && session.passport && session.passport.user) { ;({ user } = session.passport) - } else if ((session != null ? session.user : undefined) != null) { + } else if (session && session.user) { ;({ user } = session) } else { user = { _id: 'anonymous-user' } } client.on('joinProject', function (data, callback) { - if (data == null) { - data = {} - } + data = data || {} if (typeof callback !== 'function') { return Router._handleInvalidArguments( client, @@ -194,18 +170,18 @@ module.exports = Router = { if (data.anonymousAccessToken) { user.anonymousAccessToken = data.anonymousAccessToken } - return WebsocketController.joinProject( + WebsocketController.joinProject( client, user, data.project_id, function (err, ...args) { - if (err != null) { - return Router._handleError(callback, err, client, 'joinProject', { + if (err) { + Router._handleError(callback, err, client, 'joinProject', { project_id: data.project_id, - user_id: user != null ? user.id : undefined + user_id: user._id }) } else { - return callback(null, ...Array.from(args)) + callback(null, ...args) } } ) @@ -213,19 +189,11 @@ module.exports = Router = { client.on('disconnect', function () { metrics.inc('socket-io.disconnect') - metrics.gauge( - 'socket-io.clients', - __guard__(io.sockets.clients(), (x3) => x3.length) - 1 - ) + metrics.gauge('socket-io.clients', io.sockets.clients().length) - return WebsocketController.leaveProject(io, client, function (err) { - if (err != null) { - return Router._handleError( - function () {}, - err, - client, - 'leaveProject' - ) + WebsocketController.leaveProject(io, client, function (err) { + if (err) { + Router._handleError(function () {}, err, client, 'leaveProject') } }) }) @@ -263,19 +231,19 @@ module.exports = Router = { return Router._handleInvalidArguments(client, 'joinDoc', arguments) } - return WebsocketController.joinDoc( + WebsocketController.joinDoc( client, doc_id, fromVersion, options, function (err, ...args) { - if (err != null) { - return Router._handleError(callback, err, client, 'joinDoc', { + if (err) { + Router._handleError(callback, err, client, 'joinDoc', { doc_id, fromVersion }) } else { - return callback(null, ...Array.from(args)) + callback(null, ...args) } } ) @@ -286,22 +254,16 @@ module.exports = Router = { return Router._handleInvalidArguments(client, 'leaveDoc', arguments) } - return WebsocketController.leaveDoc(client, doc_id, function ( - err, - ...args - ) { - if (err != null) { - return Router._handleError(callback, err, client, 'leaveDoc') + WebsocketController.leaveDoc(client, doc_id, function (err, ...args) { + if (err) { + Router._handleError(callback, err, client, 'leaveDoc') } else { - return callback(null, ...Array.from(args)) + callback(null, ...args) } }) }) client.on('clientTracking.getConnectedUsers', function (callback) { - if (callback == null) { - callback = function (error, users) {} - } if (typeof callback !== 'function') { return Router._handleInvalidArguments( client, @@ -310,19 +272,16 @@ module.exports = Router = { ) } - return WebsocketController.getConnectedUsers(client, function ( - err, - users - ) { - if (err != null) { - return Router._handleError( + WebsocketController.getConnectedUsers(client, function (err, users) { + if (err) { + Router._handleError( callback, err, client, 'clientTracking.getConnectedUsers' ) } else { - return callback(null, users) + callback(null, users) } }) }) @@ -331,8 +290,8 @@ module.exports = Router = { cursorData, callback ) { - if (callback == null) { - callback = function (error) {} + if (!callback) { + callback = function () {} } if (typeof callback !== 'function') { return Router._handleInvalidArguments( @@ -342,28 +301,23 @@ module.exports = Router = { ) } - return WebsocketController.updateClientPosition( - client, - cursorData, - function (err) { - if (err != null) { - return Router._handleError( - callback, - err, - client, - 'clientTracking.updatePosition' - ) - } else { - return callback() - } + WebsocketController.updateClientPosition(client, cursorData, function ( + err + ) { + if (err) { + Router._handleError( + callback, + err, + client, + 'clientTracking.updatePosition' + ) + } else { + callback() } - ) + }) }) - return client.on('applyOtUpdate', function (doc_id, update, callback) { - if (callback == null) { - callback = function (error) {} - } + client.on('applyOtUpdate', function (doc_id, update, callback) { if (typeof callback !== 'function') { return Router._handleInvalidArguments( client, @@ -372,31 +326,19 @@ module.exports = Router = { ) } - return WebsocketController.applyOtUpdate( - client, - doc_id, - update, - function (err) { - if (err != null) { - return Router._handleError( - callback, - err, - client, - 'applyOtUpdate', - { doc_id, update } - ) - } else { - return callback() - } + WebsocketController.applyOtUpdate(client, doc_id, update, function ( + err + ) { + if (err) { + Router._handleError(callback, err, client, 'applyOtUpdate', { + doc_id, + update + }) + } else { + callback() } - ) + }) }) }) } } - -function __guard__(value, transform) { - return typeof value !== 'undefined' && value !== null - ? transform(value) - : undefined -} diff --git a/services/real-time/app/js/SafeJsonParse.js b/services/real-time/app/js/SafeJsonParse.js index 6e2e287853..982fd9d424 100644 --- a/services/real-time/app/js/SafeJsonParse.js +++ b/services/real-time/app/js/SafeJsonParse.js @@ -1,23 +1,8 @@ -/* eslint-disable - handle-callback-err, -*/ -// TODO: This file was created by bulk-decaffeinate. -// Fix any style issues and re-enable lint. -/* - * decaffeinate suggestions: - * DS102: Remove unnecessary code created because of implicit returns - * DS207: Consider shorter variations of null checks - * Full docs: https://github.com/decaffeinate/decaffeinate/blob/master/docs/suggestions.md - */ const Settings = require('settings-sharelatex') const logger = require('logger-sharelatex') module.exports = { parse(data, callback) { - let parsed - if (callback == null) { - callback = function (error, parsed) {} - } if (data.length > Settings.maxUpdateSize) { logger.error( { head: data.slice(0, 1024), length: data.length }, @@ -25,11 +10,12 @@ module.exports = { ) return callback(new Error('data too large to parse')) } + let parsed try { parsed = JSON.parse(data) } catch (e) { return callback(e) } - return callback(null, parsed) + callback(null, parsed) } } diff --git a/services/real-time/app/js/SessionSockets.js b/services/real-time/app/js/SessionSockets.js index b01920dfa7..4ade959829 100644 --- a/services/real-time/app/js/SessionSockets.js +++ b/services/real-time/app/js/SessionSockets.js @@ -1,34 +1,28 @@ -// TODO: This file was created by bulk-decaffeinate. -// Sanity-check the conversion and remove this comment. -/* - * decaffeinate suggestions: - * DS102: Remove unnecessary code created because of implicit returns - * Full docs: https://github.com/decaffeinate/decaffeinate/blob/master/docs/suggestions.md - */ const { EventEmitter } = require('events') module.exports = function (io, sessionStore, cookieParser, cookieName) { const missingSessionError = new Error('could not look up session by key') const sessionSockets = new EventEmitter() - const next = (error, socket, session) => + function next(error, socket, session) { sessionSockets.emit('connection', error, socket, session) + } io.on('connection', function (socket) { const req = socket.handshake - return cookieParser(req, {}, function () { + cookieParser(req, {}, function () { const sessionId = req.signedCookies && req.signedCookies[cookieName] if (!sessionId) { return next(missingSessionError, socket) } - return sessionStore.get(sessionId, function (error, session) { + sessionStore.get(sessionId, function (error, session) { if (error) { return next(error, socket) } if (!session) { return next(missingSessionError, socket) } - return next(null, socket, session) + next(null, socket, session) }) }) }) diff --git a/services/real-time/app/js/WebApiManager.js b/services/real-time/app/js/WebApiManager.js index 266135333a..62020a98ab 100644 --- a/services/real-time/app/js/WebApiManager.js +++ b/services/real-time/app/js/WebApiManager.js @@ -1,35 +1,21 @@ /* eslint-disable camelcase, - handle-callback-err, - no-unused-vars, */ -// TODO: This file was created by bulk-decaffeinate. -// Fix any style issues and re-enable lint. -/* - * decaffeinate suggestions: - * DS102: Remove unnecessary code created because of implicit returns - * DS207: Consider shorter variations of null checks - * Full docs: https://github.com/decaffeinate/decaffeinate/blob/master/docs/suggestions.md - */ -let WebApiManager const request = require('request') const settings = require('settings-sharelatex') const logger = require('logger-sharelatex') const { CodedError } = require('./Errors') -module.exports = WebApiManager = { +module.exports = { joinProject(project_id, user, callback) { - if (callback == null) { - callback = function (error, project, privilegeLevel, isRestrictedUser) {} - } const user_id = user._id logger.log({ project_id, user_id }, 'sending join project request to web') const url = `${settings.apis.web.url}/project/${project_id}/join` const headers = {} - if (user.anonymousAccessToken != null) { + if (user.anonymousAccessToken) { headers['x-sl-anonymous-access-token'] = user.anonymousAccessToken } - return request.post( + request.post( { url, qs: { user_id }, @@ -43,15 +29,12 @@ module.exports = WebApiManager = { headers }, function (error, response, data) { - let err - if (error != null) { + if (error) { return callback(error) } + let err if (response.statusCode >= 200 && response.statusCode < 300) { - if ( - data == null || - (data != null ? data.project : undefined) == null - ) { + if (!(data && data.project)) { err = new Error('no data returned from joinProject request') logger.error( { err, project_id, user_id }, @@ -59,7 +42,7 @@ module.exports = WebApiManager = { ) return callback(err) } - return callback( + callback( null, data.project, data.privilegeLevel, @@ -67,7 +50,7 @@ module.exports = WebApiManager = { ) } else if (response.statusCode === 429) { logger.log(project_id, user_id, 'rate-limit hit when joining project') - return callback( + callback( new CodedError( 'rate-limit hit when joining project', 'TooManyRequests' @@ -78,7 +61,7 @@ module.exports = WebApiManager = { `non-success status code from web: ${response.statusCode}` ) logger.error({ err, project_id, user_id }, 'error accessing web api') - return callback(err) + callback(err) } } ) diff --git a/services/real-time/app/js/WebsocketController.js b/services/real-time/app/js/WebsocketController.js index 451c514336..239372eea9 100644 --- a/services/real-time/app/js/WebsocketController.js +++ b/services/real-time/app/js/WebsocketController.js @@ -1,22 +1,8 @@ /* eslint-disable camelcase, - handle-callback-err, - no-unused-vars, */ -// TODO: This file was created by bulk-decaffeinate. -// Fix any style issues and re-enable lint. -/* - * decaffeinate suggestions: - * DS101: Remove unnecessary use of Array.from - * DS102: Remove unnecessary code created because of implicit returns - * DS103: Rewrite code to no longer use __guard__ - * DS207: Consider shorter variations of null checks - * Full docs: https://github.com/decaffeinate/decaffeinate/blob/master/docs/suggestions.md - */ -let WebsocketController const logger = require('logger-sharelatex') const metrics = require('metrics-sharelatex') -const settings = require('settings-sharelatex') const WebApiManager = require('./WebApiManager') const AuthorizationManager = require('./AuthorizationManager') const DocumentUpdaterManager = require('./DocumentUpdaterManager') @@ -24,6 +10,7 @@ const ConnectedUsersManager = require('./ConnectedUsersManager') const WebsocketLoadBalancer = require('./WebsocketLoadBalancer') const RoomManager = require('./RoomManager') +let WebsocketController module.exports = WebsocketController = { // If the protocol version changes when the client reconnects, // it will force a full refresh of the page. Useful for non-backwards @@ -31,9 +18,6 @@ module.exports = WebsocketController = { PROTOCOL_VERSION: 2, joinProject(client, user, project_id, callback) { - if (callback == null) { - callback = function (error, project, privilegeLevel, protocolVersion) {} - } if (client.disconnected) { metrics.inc('editor.join-project.disconnected', 1, { status: 'immediately' @@ -41,19 +25,19 @@ module.exports = WebsocketController = { return callback() } - const user_id = user != null ? user._id : undefined + const user_id = user._id logger.log( { user_id, project_id, client_id: client.id }, 'user joining project' ) metrics.inc('editor.join-project') - return WebApiManager.joinProject(project_id, user, function ( + WebApiManager.joinProject(project_id, user, function ( error, project, privilegeLevel, isRestrictedUser ) { - if (error != null) { + if (error) { return callback(error) } if (client.disconnected) { @@ -63,7 +47,7 @@ module.exports = WebsocketController = { return callback() } - if (!privilegeLevel || privilegeLevel === '') { + if (!privilegeLevel) { const err = new Error('not authorized') logger.warn( { err, project_id, user_id, client_id: client.id }, @@ -76,16 +60,13 @@ module.exports = WebsocketController = { client.ol_context.privilege_level = privilegeLevel client.ol_context.user_id = user_id client.ol_context.project_id = project_id - client.ol_context.owner_id = __guard__( - project != null ? project.owner : undefined, - (x) => x._id - ) - client.ol_context.first_name = user != null ? user.first_name : undefined - client.ol_context.last_name = user != null ? user.last_name : undefined - client.ol_context.email = user != null ? user.email : undefined + client.ol_context.owner_id = project.owner && project.owner._id + client.ol_context.first_name = user.first_name + client.ol_context.last_name = user.last_name + client.ol_context.email = user.email client.ol_context.connected_time = new Date() - client.ol_context.signup_date = user != null ? user.signUpDate : undefined - client.ol_context.login_count = user != null ? user.loginCount : undefined + client.ol_context.signup_date = user.signUpDate + client.ol_context.login_count = user.loginCount client.ol_context.is_restricted_user = !!isRestrictedUser RoomManager.joinProject(client, project_id, function (err) { @@ -96,7 +77,7 @@ module.exports = WebsocketController = { { user_id, project_id, client_id: client.id }, 'user joined project' ) - return callback( + callback( null, project, privilegeLevel, @@ -105,7 +86,7 @@ module.exports = WebsocketController = { }) // No need to block for setting the user as connected in the cursor tracking - return ConnectedUsersManager.updateUserPosition( + ConnectedUsersManager.updateUserPosition( project_id, client.publicId, user, @@ -120,9 +101,6 @@ module.exports = WebsocketController = { // is determined by FLUSH_IF_EMPTY_DELAY. FLUSH_IF_EMPTY_DELAY: 500, // ms leaveProject(io, client, callback) { - if (callback == null) { - callback = function (error) {} - } const { project_id, user_id } = client.ol_context if (!project_id) { return callback() @@ -144,8 +122,8 @@ module.exports = WebsocketController = { project_id, client.publicId, function (err) { - if (err != null) { - return logger.error( + if (err) { + logger.error( { err, project_id, user_id, client_id: client.id }, 'error marking client as disconnected' ) @@ -154,15 +132,15 @@ module.exports = WebsocketController = { ) RoomManager.leaveProjectAndDocs(client) - return setTimeout(function () { + setTimeout(function () { const remainingClients = io.sockets.clients(project_id) if (remainingClients.length === 0) { // Flush project in the background DocumentUpdaterManager.flushProjectToMongoAndDelete( project_id, function (err) { - if (err != null) { - return logger.error( + if (err) { + logger.error( { err, project_id, user_id, client_id: client.id }, 'error flushing to doc updater after leaving project' ) @@ -170,17 +148,11 @@ module.exports = WebsocketController = { } ) } - return callback() + callback() }, WebsocketController.FLUSH_IF_EMPTY_DELAY) }, joinDoc(client, doc_id, fromVersion, options, callback) { - if (fromVersion == null) { - fromVersion = -1 - } - if (callback == null) { - callback = function (error, doclines, version, ops, ranges) {} - } if (client.disconnected) { metrics.inc('editor.join-doc.disconnected', 1, { status: 'immediately' }) return callback() @@ -188,7 +160,7 @@ module.exports = WebsocketController = { metrics.inc('editor.join-doc') const { project_id, user_id, is_restricted_user } = client.ol_context - if (project_id == null) { + if (!project_id) { return callback(new Error('no project_id found on client')) } logger.log( @@ -196,16 +168,14 @@ module.exports = WebsocketController = { 'client joining doc' ) - return AuthorizationManager.assertClientCanViewProject(client, function ( - error - ) { - if (error != null) { + AuthorizationManager.assertClientCanViewProject(client, function (error) { + if (error) { return callback(error) } // ensure the per-doc applied-ops channel is subscribed before sending the // doc to the client, so that no events are missed. - return RoomManager.joinDoc(client, doc_id, function (error) { - if (error != null) { + RoomManager.joinDoc(client, doc_id, function (error) { + if (error) { return callback(error) } if (client.disconnected) { @@ -216,13 +186,12 @@ module.exports = WebsocketController = { return callback() } - return DocumentUpdaterManager.getDocument( + DocumentUpdaterManager.getDocument( project_id, doc_id, fromVersion, function (error, lines, version, ranges, ops) { - let err - if (error != null) { + if (error) { return callback(error) } if (client.disconnected) { @@ -233,10 +202,7 @@ module.exports = WebsocketController = { return callback() } - if ( - is_restricted_user && - (ranges != null ? ranges.comments : undefined) != null - ) { + if (is_restricted_user && ranges && ranges.comments) { ranges.comments = [] } @@ -245,11 +211,10 @@ module.exports = WebsocketController = { const encodeForWebsockets = (text) => unescape(encodeURIComponent(text)) const escapedLines = [] - for (let line of Array.from(lines)) { + for (let line of lines) { try { line = encodeForWebsockets(line) - } catch (error1) { - err = error1 + } catch (err) { logger.err( { err, @@ -267,25 +232,20 @@ module.exports = WebsocketController = { } if (options.encodeRanges) { try { - for (const comment of Array.from( - (ranges != null ? ranges.comments : undefined) || [] - )) { - if (comment.op.c != null) { + for (const comment of (ranges && ranges.comments) || []) { + if (comment.op.c) { comment.op.c = encodeForWebsockets(comment.op.c) } } - for (const change of Array.from( - (ranges != null ? ranges.changes : undefined) || [] - )) { - if (change.op.i != null) { + for (const change of (ranges && ranges.changes) || []) { + if (change.op.i) { change.op.i = encodeForWebsockets(change.op.i) } - if (change.op.d != null) { + if (change.op.d) { change.op.d = encodeForWebsockets(change.op.d) } } - } catch (error2) { - err = error2 + } catch (err) { logger.err( { err, @@ -301,7 +261,7 @@ module.exports = WebsocketController = { } } - AuthorizationManager.addAccessToDoc(client, doc_id) + AuthorizationManager.addAccessToDoc(client, doc_id, () => {}) logger.log( { user_id, @@ -312,7 +272,7 @@ module.exports = WebsocketController = { }, 'client joined doc' ) - return callback(null, escapedLines, version, ops, ranges) + callback(null, escapedLines, version, ops, ranges) } ) }) @@ -321,9 +281,6 @@ module.exports = WebsocketController = { leaveDoc(client, doc_id, callback) { // client may have disconnected, but we have to cleanup internal state. - if (callback == null) { - callback = function (error) {} - } metrics.inc('editor.leave-doc') const { project_id, user_id } = client.ol_context logger.log( @@ -335,12 +292,9 @@ module.exports = WebsocketController = { // the connection is per-project, we continue to allow access // after the initial joinDoc since we know they are already authorised. // # AuthorizationManager.removeAccessToDoc client, doc_id - return callback() + callback() }, updateClientPosition(client, cursorData, callback) { - if (callback == null) { - callback = function (error) {} - } if (client.disconnected) { // do not create a ghost entry in redis return callback() @@ -359,11 +313,11 @@ module.exports = WebsocketController = { 'updating client position' ) - return AuthorizationManager.assertClientCanViewProjectAndDoc( + AuthorizationManager.assertClientCanViewProjectAndDoc( client, cursorData.doc_id, function (error) { - if (error != null) { + if (error) { logger.warn( { err: error, client_id: client.id, project_id, user_id }, "silently ignoring unauthorized updateClientPosition. Client likely hasn't called joinProject yet." @@ -371,10 +325,10 @@ module.exports = WebsocketController = { return callback() } cursorData.id = client.publicId - if (user_id != null) { + if (user_id) { cursorData.user_id = user_id } - if (email != null) { + if (email) { cursorData.email = email } // Don't store anonymous users in redis to avoid influx @@ -404,7 +358,7 @@ module.exports = WebsocketController = { callback ) } - return WebsocketLoadBalancer.emitToRoom( + WebsocketLoadBalancer.emitToRoom( project_id, 'clientTracking.clientUpdated', cursorData @@ -415,9 +369,6 @@ module.exports = WebsocketController = { CLIENT_REFRESH_DELAY: 1000, getConnectedUsers(client, callback) { - if (callback == null) { - callback = function (error, users) {} - } if (client.disconnected) { // they are not interested anymore, skip the redis lookups return callback() @@ -428,34 +379,32 @@ module.exports = WebsocketController = { if (is_restricted_user) { return callback(null, []) } - if (project_id == null) { + if (!project_id) { return callback(new Error('no project_id found on client')) } logger.log( { user_id, project_id, client_id: client.id }, 'getting connected users' ) - return AuthorizationManager.assertClientCanViewProject(client, function ( - error - ) { - if (error != null) { + AuthorizationManager.assertClientCanViewProject(client, function (error) { + if (error) { return callback(error) } WebsocketLoadBalancer.emitToRoom(project_id, 'clientTracking.refresh') - return setTimeout( + setTimeout( () => ConnectedUsersManager.getConnectedUsers(project_id, function ( error, users ) { - if (error != null) { + if (error) { return callback(error) } - callback(null, users) - return logger.log( + logger.log( { user_id, project_id, client_id: client.id }, 'got connected users' ) + callback(null, users) }), WebsocketController.CLIENT_REFRESH_DELAY ) @@ -464,20 +413,17 @@ module.exports = WebsocketController = { applyOtUpdate(client, doc_id, update, callback) { // client may have disconnected, but we can submit their update to doc-updater anyways. - if (callback == null) { - callback = function (error) {} - } const { user_id, project_id } = client.ol_context - if (project_id == null) { + if (!project_id) { return callback(new Error('no project_id found on client')) } - return WebsocketController._assertClientCanApplyUpdate( + WebsocketController._assertClientCanApplyUpdate( client, doc_id, update, function (error) { - if (error != null) { + if (error) { logger.warn( { err: error, doc_id, client_id: client.id, version: update.v }, 'client is not authorized to make update' @@ -508,15 +454,12 @@ module.exports = WebsocketController = { 'sending update to doc updater' ) - return DocumentUpdaterManager.queueChange( + DocumentUpdaterManager.queueChange( project_id, doc_id, update, function (error) { - if ( - (error != null ? error.message : undefined) === - 'update is too large' - ) { + if ((error && error.message) === 'update is too large') { metrics.inc('update_too_large') const { updateSize } = error logger.warn( @@ -541,12 +484,12 @@ module.exports = WebsocketController = { }) } client.emit('otUpdateError', message.error, message) - return client.disconnect() + client.disconnect() }, 100) return } - if (error != null) { + if (error) { logger.error( { err: error, @@ -559,7 +502,7 @@ module.exports = WebsocketController = { ) client.disconnect() } - return callback(error) + callback(error) } ) } @@ -567,43 +510,37 @@ module.exports = WebsocketController = { }, _assertClientCanApplyUpdate(client, doc_id, update, callback) { - return AuthorizationManager.assertClientCanEditProjectAndDoc( + AuthorizationManager.assertClientCanEditProjectAndDoc( client, doc_id, function (error) { - if (error != null) { - if ( - error.message === 'not authorized' && - WebsocketController._isCommentUpdate(update) - ) { - // This might be a comment op, which we only need read-only priveleges for - return AuthorizationManager.assertClientCanViewProjectAndDoc( - client, - doc_id, - callback - ) - } else { - return callback(error) - } - } else { - return callback(null) + if ( + error && + error.message === 'not authorized' && + WebsocketController._isCommentUpdate(update) + ) { + // This might be a comment op, which we only need read-only priveleges for + AuthorizationManager.assertClientCanViewProjectAndDoc( + client, + doc_id, + callback + ) + return } + callback(error) } ) }, _isCommentUpdate(update) { - for (const op of Array.from(update.op)) { - if (op.c == null) { + if (!(update && update.op instanceof Array)) { + return false + } + for (const op of update.op) { + if (!op.c) { return false } } return true } } - -function __guard__(value, transform) { - return typeof value !== 'undefined' && value !== null - ? transform(value) - : undefined -} diff --git a/services/real-time/app/js/WebsocketLoadBalancer.js b/services/real-time/app/js/WebsocketLoadBalancer.js index 2719921f10..ee6ba4d335 100644 --- a/services/real-time/app/js/WebsocketLoadBalancer.js +++ b/services/real-time/app/js/WebsocketLoadBalancer.js @@ -1,17 +1,6 @@ /* eslint-disable camelcase, */ -// TODO: This file was created by bulk-decaffeinate. -// Fix any style issues and re-enable lint. -/* - * decaffeinate suggestions: - * DS101: Remove unnecessary use of Array.from - * DS102: Remove unnecessary code created because of implicit returns - * DS205: Consider reworking code to avoid use of IIFEs - * DS207: Consider shorter variations of null checks - * Full docs: https://github.com/decaffeinate/decaffeinate/blob/master/docs/suggestions.md - */ -let WebsocketLoadBalancer const Settings = require('settings-sharelatex') const logger = require('logger-sharelatex') const RedisClientManager = require('./RedisClientManager') @@ -33,12 +22,13 @@ const RESTRICTED_USER_MESSAGE_TYPE_PASS_LIST = [ 'removeEntity' ] +let WebsocketLoadBalancer module.exports = WebsocketLoadBalancer = { rclientPubList: RedisClientManager.createClientList(Settings.redis.pubsub), rclientSubList: RedisClientManager.createClientList(Settings.redis.pubsub), emitToRoom(room_id, message, ...payload) { - if (room_id == null) { + if (!room_id) { logger.warn( { message, payload }, 'no room_id provided, ignoring emitToRoom' @@ -55,99 +45,78 @@ module.exports = WebsocketLoadBalancer = { 'emitting to room' ) - return Array.from(this.rclientPubList).map((rclientPub) => + this.rclientPubList.map((rclientPub) => ChannelManager.publish(rclientPub, 'editor-events', room_id, data) ) }, emitToAll(message, ...payload) { - return this.emitToRoom('all', message, ...Array.from(payload)) + this.emitToRoom('all', message, ...payload) }, listenForEditorEvents(io) { - logger.log( - { rclients: this.rclientPubList.length }, - 'publishing editor events' - ) logger.log( { rclients: this.rclientSubList.length }, 'listening for editor events' ) - for (const rclientSub of Array.from(this.rclientSubList)) { + for (const rclientSub of this.rclientSubList) { rclientSub.subscribe('editor-events') rclientSub.on('message', function (channel, message) { if (Settings.debugEvents > 0) { EventLogger.debugEvent(channel, message) } - return WebsocketLoadBalancer._processEditorEvent(io, channel, message) + WebsocketLoadBalancer._processEditorEvent(io, channel, message) }) } - return this.handleRoomUpdates(this.rclientSubList) + this.handleRoomUpdates(this.rclientSubList) }, handleRoomUpdates(rclientSubList) { const roomEvents = RoomManager.eventSource() roomEvents.on('project-active', function (project_id) { - const subscribePromises = Array.from(rclientSubList).map((rclient) => + const subscribePromises = rclientSubList.map((rclient) => ChannelManager.subscribe(rclient, 'editor-events', project_id) ) - return RoomManager.emitOnCompletion( + RoomManager.emitOnCompletion( subscribePromises, `project-subscribed-${project_id}` ) }) - return roomEvents.on('project-empty', (project_id) => - Array.from(rclientSubList).map((rclient) => + roomEvents.on('project-empty', (project_id) => + rclientSubList.map((rclient) => ChannelManager.unsubscribe(rclient, 'editor-events', project_id) ) ) }, _processEditorEvent(io, channel, message) { - return SafeJsonParse.parse(message, function (error, message) { - let clientList - let client - if (error != null) { + SafeJsonParse.parse(message, function (error, message) { + if (error) { logger.error({ err: error, channel }, 'error parsing JSON') return } if (message.room_id === 'all') { - return io.sockets.emit(message.message, ...Array.from(message.payload)) + io.sockets.emit(message.message, ...message.payload) } else if ( message.message === 'clientTracking.refresh' && - message.room_id != null + message.room_id ) { - clientList = io.sockets.clients(message.room_id) + const clientList = io.sockets.clients(message.room_id) logger.log( { channel, message: message.message, room_id: message.room_id, message_id: message._id, - socketIoClients: (() => { - const result = [] - for (client of Array.from(clientList)) { - result.push(client.id) - } - return result - })() + socketIoClients: clientList.map((client) => client.id) }, 'refreshing client list' ) - return (() => { - const result1 = [] - for (client of Array.from(clientList)) { - result1.push( - ConnectedUsersManager.refreshClient( - message.room_id, - client.publicId - ) - ) - } - return result1 - })() - } else if (message.room_id != null) { - if (message._id != null && Settings.checkEventOrder) { + for (const client of clientList) { + ConnectedUsersManager.refreshClient(message.room_id, client.publicId) + } + } else if (message.room_id) { + if (message._id && Settings.checkEventOrder) { const status = EventLogger.checkEventOrder( 'editor-events', message._id, @@ -158,12 +127,12 @@ module.exports = WebsocketLoadBalancer = { } } - const is_restricted_message = !Array.from( - RESTRICTED_USER_MESSAGE_TYPE_PASS_LIST - ).includes(message.message) + const is_restricted_message = !RESTRICTED_USER_MESSAGE_TYPE_PASS_LIST.includes( + message.message + ) // send messages only to unique clients (due to duplicate entries in io.sockets.clients) - clientList = io.sockets + const clientList = io.sockets .clients(message.room_id) .filter( (client) => @@ -180,37 +149,23 @@ module.exports = WebsocketLoadBalancer = { message: message.message, room_id: message.room_id, message_id: message._id, - socketIoClients: (() => { - const result2 = [] - for (client of Array.from(clientList)) { - result2.push(client.id) - } - return result2 - })() + socketIoClients: clientList.map((client) => client.id) }, 'distributing event to clients' ) - const seen = {} - return (() => { - const result3 = [] - for (client of Array.from(clientList)) { - if (!seen[client.id]) { - seen[client.id] = true - result3.push( - client.emit(message.message, ...Array.from(message.payload)) - ) - } else { - result3.push(undefined) - } + const seen = new Map() + for (const client of clientList) { + if (!seen.has(client.id)) { + seen.set(client.id, true) + client.emit(message.message, ...message.payload) } - return result3 - })() - } else if (message.health_check != null) { + } + } else if (message.health_check) { logger.debug( { message }, 'got health check message in editor events channel' ) - return HealthCheckManager.check(channel, message.key) + HealthCheckManager.check(channel, message.key) } }) } From 9fff03bca5748afaaabf065cd89e6db5f64b8d41 Mon Sep 17 00:00:00 2001 From: Jakob Ackermann Date: Mon, 6 Jul 2020 14:27:46 +0100 Subject: [PATCH 397/491] [misc] optionally expose the hostname in the 'debug.getHostname' rpc --- services/real-time/app/js/Router.js | 15 +++++++++++++++ services/real-time/config/settings.defaults.js | 3 +++ 2 files changed, 18 insertions(+) diff --git a/services/real-time/app/js/Router.js b/services/real-time/app/js/Router.js index 02c751c22f..0b4b3f00d7 100644 --- a/services/real-time/app/js/Router.js +++ b/services/real-time/app/js/Router.js @@ -21,6 +21,8 @@ const httpAuth = basicAuth(function (user, pass) { return isValid }) +const HOSTNAME = require('os').hostname() + let Router module.exports = Router = { _handleError(callback, error, client, method, attrs) { @@ -157,6 +159,19 @@ module.exports = Router = { user = { _id: 'anonymous-user' } } + if (settings.exposeHostname) { + client.on('debug.getHostname', function (callback) { + if (typeof callback !== 'function') { + return Router._handleInvalidArguments( + client, + 'debug.getHostname', + arguments + ) + } + callback(HOSTNAME) + }) + } + client.on('joinProject', function (data, callback) { data = data || {} if (typeof callback !== 'function') { diff --git a/services/real-time/config/settings.defaults.js b/services/real-time/config/settings.defaults.js index 1e4ec6fe72..cbf6d3be82 100644 --- a/services/real-time/config/settings.defaults.js +++ b/services/real-time/config/settings.defaults.js @@ -109,6 +109,9 @@ const settings = { cookieName: process.env.COOKIE_NAME || 'sharelatex.sid', + // Expose the hostname in the `debug.getHostname` rpc + exposeHostname: process.env.EXPOSE_HOSTNAME === 'true', + max_doc_length: 2 * 1024 * 1024, // 2mb // combine From ca4168ce90a7b02463cfbb4f8f9947eff9a01dbc Mon Sep 17 00:00:00 2001 From: Jakob Ackermann Date: Wed, 8 Jul 2020 15:07:19 +0100 Subject: [PATCH 398/491] [misc] use overleaf/socket.io and overleaf/socket.io-client forks --- services/real-time/app.js | 2 - services/real-time/package-lock.json | 23 ++------- services/real-time/package.json | 4 +- services/real-time/socket.io.patch.js | 74 --------------------------- 4 files changed, 7 insertions(+), 96 deletions(-) delete mode 100644 services/real-time/socket.io.patch.js diff --git a/services/real-time/app.js b/services/real-time/app.js index 1580f18027..4499b6adf0 100644 --- a/services/real-time/app.js +++ b/services/real-time/app.js @@ -23,8 +23,6 @@ const CookieParser = require('cookie-parser') const DrainManager = require('./app/js/DrainManager') const HealthCheckManager = require('./app/js/HealthCheckManager') -// work around frame handler bug in socket.io v0.9.16 -require('./socket.io.patch.js') // Set up socket.io server const app = express() diff --git a/services/real-time/package-lock.json b/services/real-time/package-lock.json index 616f0edea9..399b980143 100644 --- a/services/real-time/package-lock.json +++ b/services/real-time/package-lock.json @@ -4530,14 +4530,13 @@ } }, "socket.io": { - "version": "0.9.19", - "resolved": "https://registry.npmjs.org/socket.io/-/socket.io-0.9.19.tgz", - "integrity": "sha512-UPdVIGPBPmCibzIP2rAjXuiPTI2gPs6kiu4P7njH6WAK7wiOlozNG62ohohCNOycx+Dztd4vRNXxq8alIOEtfA==", + "version": "https://github.com/overleaf/socket.io/archive/0.9.19-overleaf-2.tar.gz", + "integrity": "sha512-BVxF8Wz4FTj2hxiBtujKQUAihdVzxjSaJ++k/wr7pJfAt30kmyOXLkfyvFDzZISQ9SyDa2B4nBYJP3+MKBpaAg==", "requires": { "base64id": "0.1.0", "policyfile": "0.0.4", "redis": "0.7.3", - "socket.io-client": "0.9.16" + "socket.io-client": "https://github.com/overleaf/socket.io-client/archive/0.9.17-overleaf-1.tar.gz" }, "dependencies": { "redis": { @@ -4545,24 +4544,12 @@ "resolved": "https://registry.npmjs.org/redis/-/redis-0.7.3.tgz", "integrity": "sha512-0Pgb0jOLfn6eREtEIRn/ifyZJjl2H+wUY4F/Pe7T4UhmoSrZ/1HU5ZqiBpDk8I8Wbyv2N5DpXKzbEtMj3drprg==", "optional": true - }, - "socket.io-client": { - "version": "0.9.16", - "resolved": "https://registry.npmjs.org/socket.io-client/-/socket.io-client-0.9.16.tgz", - "integrity": "sha512-wSM7PKJkzpGqUAo6d6SAn+ph4xeQJ6nzyDULRJAX1G7e6Xm0wNuMh2RpNvwXrHMzoV9Or5hti7LINiQAm1H2yA==", - "requires": { - "active-x-obfuscator": "0.0.1", - "uglify-js": "1.2.5", - "ws": "0.4.x", - "xmlhttprequest": "1.4.2" - } } } }, "socket.io-client": { - "version": "0.9.17", - "resolved": "https://registry.npmjs.org/socket.io-client/-/socket.io-client-0.9.17.tgz", - "integrity": "sha512-gKV451FUZPLeJmA8vPfvqRZctAzWNOlaB0C06MeDFmrGquDNMllnpXp+1+4QS2NaZvcycoJHVt72R5uNaLCIBg==", + "version": "https://github.com/overleaf/socket.io-client/archive/0.9.17-overleaf-1.tar.gz", + "integrity": "sha512-MchfS0GCu0BVbJXRk+HfHme5jHPWMDrNifbOhXZHJBtF03dLfCJcMPLYMQBtBGWQrrZsqXI0O9P3BO3hu0cPLA==", "requires": { "active-x-obfuscator": "0.0.1", "uglify-js": "1.2.5", diff --git a/services/real-time/package.json b/services/real-time/package.json index 5d9f58079e..cfb3e47d45 100644 --- a/services/real-time/package.json +++ b/services/real-time/package.json @@ -32,8 +32,8 @@ "redis-sharelatex": "^1.0.12", "request": "^2.88.2", "settings-sharelatex": "^1.1.0", - "socket.io": "0.9.19", - "socket.io-client": "^0.9.16" + "socket.io": "https://github.com/overleaf/socket.io/archive/0.9.19-overleaf-2.tar.gz", + "socket.io-client": "https://github.com/overleaf/socket.io-client/archive/0.9.17-overleaf-1.tar.gz" }, "devDependencies": { "bunyan": "~0.22.3", diff --git a/services/real-time/socket.io.patch.js b/services/real-time/socket.io.patch.js deleted file mode 100644 index 5852a4f266..0000000000 --- a/services/real-time/socket.io.patch.js +++ /dev/null @@ -1,74 +0,0 @@ -// EventEmitter has been removed from process in node >= 7 -// https://github.com/nodejs/node/commit/62b544290a075fe38e233887a06c408ba25a1c71 -/* - A socket.io dependency expects the EventEmitter to be available at - `process.EventEmitter`. - See this trace: - --- - - /app/node_modules/policyfile/lib/server.js:254 - Object.keys(process.EventEmitter.prototype).forEach(function proxy (key){ - ^ - - TypeError: Cannot read property 'prototype' of undefined - at Object. (/app/node_modules/policyfile/lib/server.js:254:34) - */ -if (process.versions.node.split('.')[0] >= 7) { - // eslint-disable-next-line node/no-deprecated-api - process.EventEmitter = require('events') -} - -var io = require('socket.io') -const logger = require('logger-sharelatex') - -if (io.version === '0.9.16' || io.version === '0.9.19') { - logger.warn('patching socket.io hybi-16 transport frame prototype') - var transports = require('socket.io/lib/transports/websocket/hybi-16.js') - transports.prototype.frame = patchedFrameHandler - // file hybi-07-12 has the same problem but no browsers are using that protocol now -} - -function patchedFrameHandler(opcode, str) { - var dataBuffer = Buffer.from(str) - var dataLength = dataBuffer.length - var startOffset = 2 - var secondByte = dataLength - if (dataLength === 65536) { - logger.log('fixing invalid frame length in socket.io') - } - if (dataLength > 65535) { - // original code had > 65536 - startOffset = 10 - secondByte = 127 - } else if (dataLength > 125) { - startOffset = 4 - secondByte = 126 - } - var outputBuffer = Buffer.alloc(dataLength + startOffset) - outputBuffer[0] = opcode - outputBuffer[1] = secondByte - dataBuffer.copy(outputBuffer, startOffset) - switch (secondByte) { - case 126: - outputBuffer[2] = dataLength >>> 8 - outputBuffer[3] = dataLength % 256 - break - case 127: - var l = dataLength - for (var i = 1; i <= 8; ++i) { - outputBuffer[startOffset - i] = l & 0xff - l >>>= 8 - } - } - return outputBuffer -} - -const parser = require('socket.io/lib/parser') -const decodePacket = parser.decodePacket -parser.decodePacket = function (data) { - if (typeof data !== 'string') return {} - const firstColon = data.indexOf(':') - if (firstColon === -1) return {} - if (data.indexOf(':', firstColon + 1) === -1) return {} - return decodePacket(data) -} From 562375d351c7ce376085ded72d80d94bb25dd011 Mon Sep 17 00:00:00 2001 From: Jakob Ackermann Date: Wed, 22 Jul 2020 09:45:14 +0100 Subject: [PATCH 399/491] [misc] fix express deprecations --- services/real-time/app.js | 2 +- services/real-time/app/js/HttpApiController.js | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/services/real-time/app.js b/services/real-time/app.js index 4499b6adf0..c41743e597 100644 --- a/services/real-time/app.js +++ b/services/real-time/app.js @@ -68,7 +68,7 @@ app.get('/', (req, res) => res.send('real-time-sharelatex is alive')) app.get('/status', function (req, res) { if (Settings.shutDownInProgress) { - res.send(503) // Service unavailable + res.sendStatus(503) // Service unavailable } else { res.send('real-time-sharelatex is alive') } diff --git a/services/real-time/app/js/HttpApiController.js b/services/real-time/app/js/HttpApiController.js index 7b9627fd7b..dceb294310 100644 --- a/services/real-time/app/js/HttpApiController.js +++ b/services/real-time/app/js/HttpApiController.js @@ -23,7 +23,7 @@ module.exports = { req.body ) } - res.send(204) + res.sendStatus(204) }, startDrain(req, res) { @@ -32,7 +32,7 @@ module.exports = { rate = parseFloat(rate) || 0 logger.log({ rate }, 'setting client drain rate') DrainManager.startDrain(io, rate) - res.send(204) + res.sendStatus(204) }, disconnectClient(req, res, next) { From 3d92858939134d10fa2225e289219eb8d4879c96 Mon Sep 17 00:00:00 2001 From: Shane Kilkelly Date: Fri, 24 Jul 2020 10:42:46 +0100 Subject: [PATCH 400/491] Upgrade redis-sharelatex to 1.0.13 --- services/real-time/package-lock.json | 68 ++++++++++++++-------------- services/real-time/package.json | 2 +- 2 files changed, 36 insertions(+), 34 deletions(-) diff --git a/services/real-time/package-lock.json b/services/real-time/package-lock.json index 399b980143..f94d11b719 100644 --- a/services/real-time/package-lock.json +++ b/services/real-time/package-lock.json @@ -2250,6 +2250,37 @@ } } }, + "ioredis": { + "version": "4.17.3", + "resolved": "https://registry.npmjs.org/ioredis/-/ioredis-4.17.3.tgz", + "integrity": "sha512-iRvq4BOYzNFkDnSyhx7cmJNOi1x/HWYe+A4VXHBu4qpwJaGT1Mp+D2bVGJntH9K/Z/GeOM/Nprb8gB3bmitz1Q==", + "requires": { + "cluster-key-slot": "^1.1.0", + "debug": "^4.1.1", + "denque": "^1.1.0", + "lodash.defaults": "^4.2.0", + "lodash.flatten": "^4.4.0", + "redis-commands": "1.5.0", + "redis-errors": "^1.2.0", + "redis-parser": "^3.0.0", + "standard-as-callback": "^2.0.1" + }, + "dependencies": { + "debug": { + "version": "4.1.1", + "resolved": "https://registry.npmjs.org/debug/-/debug-4.1.1.tgz", + "integrity": "sha512-pYAIzeRo8J6KPEaJ0VWOh5Pzkbw/RetuzehGM7QRRX5he4fPHx2rdKMB256ehJCkX+XRQm16eZLqLNS8RSZXZw==", + "requires": { + "ms": "^2.1.1" + } + }, + "ms": { + "version": "2.1.2", + "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.2.tgz", + "integrity": "sha512-sGkPx+VjMtmA6MX27oA4FBFELFCZZ4S4XqeGOXCv68tT+jb3vk/RyaKWP0PTKyWtmLSM0b+adUTEvbs1PEaH2w==" + } + } + }, "ipaddr.js": { "version": "1.9.1", "resolved": "https://registry.npmjs.org/ipaddr.js/-/ipaddr.js-1.9.1.tgz", @@ -4075,13 +4106,13 @@ } }, "redis-sharelatex": { - "version": "1.0.12", - "resolved": "https://registry.npmjs.org/redis-sharelatex/-/redis-sharelatex-1.0.12.tgz", - "integrity": "sha512-Z+LDGaRNgZ+NiDaCC/R0N3Uy6SCtbKXqiXlvCwAbIQRSZUc69OVx/cQ3i5qDF7zeERhh+pnTd+zGs8nVfa5p+Q==", + "version": "1.0.13", + "resolved": "https://registry.npmjs.org/redis-sharelatex/-/redis-sharelatex-1.0.13.tgz", + "integrity": "sha512-sAQNofqfcMlIxzxNJF1qUspJKDM1VuuIOrGZQX9nb5JtcJ5cusa5sc+Oyb51eymPV5mZGWT3u07tKtv4jdXVIg==", "requires": { "async": "^2.5.0", "coffee-script": "1.8.0", - "ioredis": "~4.16.1", + "ioredis": "~4.17.3", "redis-sentinel": "0.1.1", "underscore": "1.7.0" }, @@ -4102,39 +4133,10 @@ "mkdirp": "~0.3.5" } }, - "debug": { - "version": "4.1.1", - "resolved": "https://registry.npmjs.org/debug/-/debug-4.1.1.tgz", - "integrity": "sha512-pYAIzeRo8J6KPEaJ0VWOh5Pzkbw/RetuzehGM7QRRX5he4fPHx2rdKMB256ehJCkX+XRQm16eZLqLNS8RSZXZw==", - "requires": { - "ms": "^2.1.1" - } - }, - "ioredis": { - "version": "4.16.3", - "resolved": "https://registry.npmjs.org/ioredis/-/ioredis-4.16.3.tgz", - "integrity": "sha512-Ejvcs2yW19Vq8AipvbtfcX3Ig8XG9EAyFOvGbhI/Q1QoVOK9ZdgY092kdOyOWIYBnPHjfjMJhU9qhsnp0i0K1w==", - "requires": { - "cluster-key-slot": "^1.1.0", - "debug": "^4.1.1", - "denque": "^1.1.0", - "lodash.defaults": "^4.2.0", - "lodash.flatten": "^4.4.0", - "redis-commands": "1.5.0", - "redis-errors": "^1.2.0", - "redis-parser": "^3.0.0", - "standard-as-callback": "^2.0.1" - } - }, "mkdirp": { "version": "0.3.5", "resolved": "https://registry.npmjs.org/mkdirp/-/mkdirp-0.3.5.tgz", "integrity": "sha512-8OCq0De/h9ZxseqzCH8Kw/Filf5pF/vMI6+BH7Lu0jXz2pqYCjTAQRolSxRIi+Ax+oCCjlxoJMP0YQ4XlrQNHg==" - }, - "ms": { - "version": "2.1.2", - "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.2.tgz", - "integrity": "sha512-sGkPx+VjMtmA6MX27oA4FBFELFCZZ4S4XqeGOXCv68tT+jb3vk/RyaKWP0PTKyWtmLSM0b+adUTEvbs1PEaH2w==" } } }, diff --git a/services/real-time/package.json b/services/real-time/package.json index cfb3e47d45..be1ebc79bb 100644 --- a/services/real-time/package.json +++ b/services/real-time/package.json @@ -29,7 +29,7 @@ "express-session": "^1.17.1", "logger-sharelatex": "^1.7.0", "metrics-sharelatex": "^2.6.2", - "redis-sharelatex": "^1.0.12", + "redis-sharelatex": "^1.0.13", "request": "^2.88.2", "settings-sharelatex": "^1.1.0", "socket.io": "https://github.com/overleaf/socket.io/archive/0.9.19-overleaf-2.tar.gz", From a91f78634b278816ce54bd69986dfa00263332b2 Mon Sep 17 00:00:00 2001 From: Jakob Ackermann Date: Thu, 30 Jul 2020 11:53:39 +0100 Subject: [PATCH 401/491] [misc] bump socket.io and socket.io-client --- services/real-time/package-lock.json | 40 ++++++++++++++-------------- services/real-time/package.json | 4 +-- 2 files changed, 22 insertions(+), 22 deletions(-) diff --git a/services/real-time/package-lock.json b/services/real-time/package-lock.json index f94d11b719..045283d72a 100644 --- a/services/real-time/package-lock.json +++ b/services/real-time/package-lock.json @@ -449,7 +449,7 @@ "active-x-obfuscator": { "version": "0.0.1", "resolved": "https://registry.npmjs.org/active-x-obfuscator/-/active-x-obfuscator-0.0.1.tgz", - "integrity": "sha512-8gdEZinfLSCfAUulETDth4ZSIDPSchiPgm5PLrXQC6BANf1YFEDrPPM2MdK2zcekMROwtM667QFuYw/H6ZV06Q==", + "integrity": "sha1-CJuJs3FF/x2ex0r2UwvlUmyuHxo=", "requires": { "zeparser": "0.0.5" } @@ -863,6 +863,11 @@ "delayed-stream": "~1.0.0" } }, + "commander": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/commander/-/commander-2.1.0.tgz", + "integrity": "sha1-0SG7roYNmZKj1Re6lvVliOR8Z4E=" + }, "common-tags": { "version": "1.8.0", "resolved": "https://registry.npmjs.org/common-tags/-/common-tags-1.8.0.tgz", @@ -3128,7 +3133,7 @@ "options": { "version": "0.0.6", "resolved": "https://registry.npmjs.org/options/-/options-0.0.6.tgz", - "integrity": "sha512-bOj3L1ypm++N+n7CEbbe473A414AB7z+amKYshRb//iuL3MpdDCLhPnw6aVTdKB9g5ZRVHIEp8eUln6L2NUStg==" + "integrity": "sha1-7CLTEoBrtT5zF3Pnza788cZDEo8=" }, "os-tmpdir": { "version": "1.0.2", @@ -3280,7 +3285,7 @@ "policyfile": { "version": "0.0.4", "resolved": "https://registry.npmjs.org/policyfile/-/policyfile-0.0.4.tgz", - "integrity": "sha512-UfDtlscNialXfmVEwEPm0t/5qtM0xPK025eYWd/ilv89hxLIhVQmt3QIzMHincLO2MBtZyww0386pt13J4aIhQ==" + "integrity": "sha1-1rgurZiueeviKOLa9ZAzEeyYLk0=" }, "prelude-ls": { "version": "1.1.2", @@ -4532,26 +4537,26 @@ } }, "socket.io": { - "version": "https://github.com/overleaf/socket.io/archive/0.9.19-overleaf-2.tar.gz", - "integrity": "sha512-BVxF8Wz4FTj2hxiBtujKQUAihdVzxjSaJ++k/wr7pJfAt30kmyOXLkfyvFDzZISQ9SyDa2B4nBYJP3+MKBpaAg==", + "version": "https://github.com/overleaf/socket.io/archive/0.9.19-overleaf-3.tar.gz", + "integrity": "sha512-x07f3i4f3/CXaTBpUT/mH4JtYf3xjZWO0tvIB43Nn8OnzeRhMaKxhMZWcLQxfPdcaAyvafwLeM06fiDpHCaIYw==", "requires": { "base64id": "0.1.0", "policyfile": "0.0.4", "redis": "0.7.3", - "socket.io-client": "https://github.com/overleaf/socket.io-client/archive/0.9.17-overleaf-1.tar.gz" + "socket.io-client": "https://github.com/overleaf/socket.io-client/archive/0.9.17-overleaf-2.tar.gz" }, "dependencies": { "redis": { "version": "0.7.3", "resolved": "https://registry.npmjs.org/redis/-/redis-0.7.3.tgz", - "integrity": "sha512-0Pgb0jOLfn6eREtEIRn/ifyZJjl2H+wUY4F/Pe7T4UhmoSrZ/1HU5ZqiBpDk8I8Wbyv2N5DpXKzbEtMj3drprg==", + "integrity": "sha1-7le3pE0l7BWU5ENl2BZfp9HUgRo=", "optional": true } } }, "socket.io-client": { - "version": "https://github.com/overleaf/socket.io-client/archive/0.9.17-overleaf-1.tar.gz", - "integrity": "sha512-MchfS0GCu0BVbJXRk+HfHme5jHPWMDrNifbOhXZHJBtF03dLfCJcMPLYMQBtBGWQrrZsqXI0O9P3BO3hu0cPLA==", + "version": "https://github.com/overleaf/socket.io-client/archive/0.9.17-overleaf-2.tar.gz", + "integrity": "sha512-u8jEVctc4nFrN1JfjLeFjSSxXxGYo6hHxYW4rhv6aeoiaWkTmdaUzaG1YkDfSwBG/TXmdHx32UJB4fClmz8IEg==", "requires": { "active-x-obfuscator": "0.0.1", "uglify-js": "1.2.5", @@ -4835,7 +4840,7 @@ "tinycolor": { "version": "0.0.1", "resolved": "https://registry.npmjs.org/tinycolor/-/tinycolor-0.0.1.tgz", - "integrity": "sha512-+CorETse1kl98xg0WAzii8DTT4ABF4R3nquhrkIbVGcw1T8JYs5Gfx9xEfGINPUZGDj9C4BmOtuKeaTtuuRolg==" + "integrity": "sha1-MgtaUtg6u1l42Bo+iH1K77FaYWQ=" }, "tmp": { "version": "0.0.33", @@ -4960,7 +4965,7 @@ "uglify-js": { "version": "1.2.5", "resolved": "https://registry.npmjs.org/uglify-js/-/uglify-js-1.2.5.tgz", - "integrity": "sha512-Ps1oQryKOcRDYuAN1tGpPWd/DIRMcdLz4p7JMxLjJiFvp+aaG01IEu0ZSoVvYUSxIkvW7k2X50BCW2InguEGlg==" + "integrity": "sha1-tULCx29477NLIAsgF3Y0Mw/3ArY=" }, "uid-safe": { "version": "2.1.5", @@ -5182,7 +5187,7 @@ "ws": { "version": "0.4.32", "resolved": "https://registry.npmjs.org/ws/-/ws-0.4.32.tgz", - "integrity": "sha512-htqsS0U9Z9lb3ITjidQkRvkLdVhQePrMeu475yEfOWkAYvJ6dSjQp1tOH6ugaddzX5b7sQjMPNtY71eTzrV/kA==", + "integrity": "sha1-eHphVEFPPJntg8V3IVOyD+sM7DI=", "requires": { "commander": "~2.1.0", "nan": "~1.0.0", @@ -5190,22 +5195,17 @@ "tinycolor": "0.x" }, "dependencies": { - "commander": { - "version": "2.1.0", - "resolved": "https://registry.npmjs.org/commander/-/commander-2.1.0.tgz", - "integrity": "sha512-J2wnb6TKniXNOtoHS8TSrG9IOQluPrsmyAJ8oCUJOBmv+uLBCyPYAZkD2jFvw2DCzIXNnISIM01NIvr35TkBMQ==" - }, "nan": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/nan/-/nan-1.0.0.tgz", - "integrity": "sha512-Wm2/nFOm2y9HtJfgOLnctGbfvF23FcQZeyUZqDD8JQG3zO5kXh3MkQKiUaA68mJiVWrOzLFkAV1u6bC8P52DJA==" + "integrity": "sha1-riT4hQgY1mL8q1rPfzuVv6oszzg=" } } }, "xmlhttprequest": { "version": "1.4.2", "resolved": "https://registry.npmjs.org/xmlhttprequest/-/xmlhttprequest-1.4.2.tgz", - "integrity": "sha512-WTsthd44hTdCRrHkdtTgbgTKIJyNDV+xiShdooFZBUstY7xk+EXMx/u5gjuUXaCiCWvtBVCHwauzml2joevB4w==" + "integrity": "sha1-AUU6HZvtHo8XL2SVu/TIxCYyFQA=" }, "y18n": { "version": "4.0.0", @@ -5307,7 +5307,7 @@ "zeparser": { "version": "0.0.5", "resolved": "https://registry.npmjs.org/zeparser/-/zeparser-0.0.5.tgz", - "integrity": "sha512-Qj4lJIRPy7hIW1zCBqwA3AW8F9uHswVoXPnotuY6uyNgbg5qGb6SJfWZi+YzD3DktbUnUoGiGZFhopbn9l1GYw==" + "integrity": "sha1-A3JlYbwmjy5URPVMZlt/1KjAKeI=" } } } diff --git a/services/real-time/package.json b/services/real-time/package.json index be1ebc79bb..821bcd2cfd 100644 --- a/services/real-time/package.json +++ b/services/real-time/package.json @@ -32,8 +32,8 @@ "redis-sharelatex": "^1.0.13", "request": "^2.88.2", "settings-sharelatex": "^1.1.0", - "socket.io": "https://github.com/overleaf/socket.io/archive/0.9.19-overleaf-2.tar.gz", - "socket.io-client": "https://github.com/overleaf/socket.io-client/archive/0.9.17-overleaf-1.tar.gz" + "socket.io": "https://github.com/overleaf/socket.io/archive/0.9.19-overleaf-3.tar.gz", + "socket.io-client": "https://github.com/overleaf/socket.io-client/archive/0.9.17-overleaf-2.tar.gz" }, "devDependencies": { "bunyan": "~0.22.3", From f4d9ae11502dab540ef65586045be9e100927d05 Mon Sep 17 00:00:00 2001 From: Jakob Ackermann Date: Tue, 4 Aug 2020 11:07:19 +0100 Subject: [PATCH 402/491] [misc] bump the dev-env to 3.3.0 --- services/real-time/.github/dependabot.yml | 17 +++ services/real-time/.npmrc | 2 + services/real-time/Dockerfile | 2 - services/real-time/Jenkinsfile | 131 ---------------------- services/real-time/Makefile | 6 +- services/real-time/buildscript.txt | 4 +- services/real-time/docker-compose.ci.yml | 2 + services/real-time/docker-compose.yml | 6 +- services/real-time/nodemon.json | 1 - services/real-time/package.json | 2 +- 10 files changed, 30 insertions(+), 143 deletions(-) create mode 100644 services/real-time/.github/dependabot.yml create mode 100644 services/real-time/.npmrc delete mode 100644 services/real-time/Jenkinsfile diff --git a/services/real-time/.github/dependabot.yml b/services/real-time/.github/dependabot.yml new file mode 100644 index 0000000000..c6f98d843d --- /dev/null +++ b/services/real-time/.github/dependabot.yml @@ -0,0 +1,17 @@ +version: 2 +updates: + - package-ecosystem: "npm" + directory: "/" + schedule: + interval: "daily" + + pull-request-branch-name: + # Separate sections of the branch name with a hyphen + # Docker images use the branch name and do not support slashes in tags + # https://github.com/overleaf/google-ops/issues/822 + # https://docs.github.com/en/github/administering-a-repository/configuration-options-for-dependency-updates#pull-request-branch-nameseparator + separator: "-" + + # Block informal upgrades -- security upgrades use a separate queue. + # https://docs.github.com/en/github/administering-a-repository/configuration-options-for-dependency-updates#open-pull-requests-limit + open-pull-requests-limit: 0 diff --git a/services/real-time/.npmrc b/services/real-time/.npmrc new file mode 100644 index 0000000000..34ecfacbc4 --- /dev/null +++ b/services/real-time/.npmrc @@ -0,0 +1,2 @@ +# block local, outside docker invocations of $ npm install +dry-run=true diff --git a/services/real-time/Dockerfile b/services/real-time/Dockerfile index b07f7117bc..78a715757d 100644 --- a/services/real-time/Dockerfile +++ b/services/real-time/Dockerfile @@ -15,8 +15,6 @@ RUN npm ci --quiet COPY . /app - - FROM base COPY --from=app /app /app diff --git a/services/real-time/Jenkinsfile b/services/real-time/Jenkinsfile deleted file mode 100644 index 4fc4f79e8a..0000000000 --- a/services/real-time/Jenkinsfile +++ /dev/null @@ -1,131 +0,0 @@ -String cron_string = BRANCH_NAME == "master" ? "@daily" : "" - -pipeline { - agent any - - environment { - GIT_PROJECT = "real-time" - JENKINS_WORKFLOW = "real-time-sharelatex" - TARGET_URL = "${env.JENKINS_URL}blue/organizations/jenkins/${JENKINS_WORKFLOW}/detail/$BRANCH_NAME/$BUILD_NUMBER/pipeline" - GIT_API_URL = "https://api.github.com/repos/overleaf/${GIT_PROJECT}/statuses/$GIT_COMMIT" - } - - triggers { - pollSCM('* * * * *') - cron(cron_string) - } - - stages { - - stage('Install') { - steps { - withCredentials([usernamePassword(credentialsId: 'GITHUB_INTEGRATION', usernameVariable: 'GH_AUTH_USERNAME', passwordVariable: 'GH_AUTH_PASSWORD')]) { - sh "curl $GIT_API_URL \ - --data '{ \ - \"state\" : \"pending\", \ - \"target_url\": \"$TARGET_URL\", \ - \"description\": \"Your build is underway\", \ - \"context\": \"ci/jenkins\" }' \ - -u $GH_AUTH_USERNAME:$GH_AUTH_PASSWORD" - } - } - } - - stage('Build') { - steps { - sh 'make build' - } - } - - stage('Linting') { - steps { - sh 'DOCKER_COMPOSE_FLAGS="-f docker-compose.ci.yml" make format' - sh 'DOCKER_COMPOSE_FLAGS="-f docker-compose.ci.yml" make lint' - } - } - - stage('Unit Tests') { - steps { - sh 'DOCKER_COMPOSE_FLAGS="-f docker-compose.ci.yml" make test_unit' - } - } - - stage('Acceptance Tests') { - steps { - sh 'DOCKER_COMPOSE_FLAGS="-f docker-compose.ci.yml" make test_acceptance' - } - } - - stage('Package and docker push') { - steps { - sh 'echo ${BUILD_NUMBER} > build_number.txt' - sh 'touch build.tar.gz' // Avoid tar warning about files changing during read - sh 'DOCKER_COMPOSE_FLAGS="-f docker-compose.ci.yml" make tar' - - withCredentials([file(credentialsId: 'gcr.io_overleaf-ops', variable: 'DOCKER_REPO_KEY_PATH')]) { - sh 'docker login -u _json_key --password-stdin https://gcr.io/overleaf-ops < ${DOCKER_REPO_KEY_PATH}' - } - sh 'DOCKER_REPO=gcr.io/overleaf-ops make publish' - sh 'docker logout https://gcr.io/overleaf-ops' - - } - } - - stage('Publish to s3') { - steps { - sh 'echo ${BRANCH_NAME}-${BUILD_NUMBER} > build_number.txt' - withAWS(credentials:'S3_CI_BUILDS_AWS_KEYS', region:"${S3_REGION_BUILD_ARTEFACTS}") { - s3Upload(file:'build.tar.gz', bucket:"${S3_BUCKET_BUILD_ARTEFACTS}", path:"${JOB_NAME}/${BUILD_NUMBER}.tar.gz") - } - withAWS(credentials:'S3_CI_BUILDS_AWS_KEYS', region:"${S3_REGION_BUILD_ARTEFACTS}") { - // The deployment process uses this file to figure out the latest build - s3Upload(file:'build_number.txt', bucket:"${S3_BUCKET_BUILD_ARTEFACTS}", path:"${JOB_NAME}/latest") - } - } - } - } - - post { - always { - sh 'DOCKER_COMPOSE_FLAGS="-f docker-compose.ci.yml" make test_clean' - sh 'make clean' - } - - success { - withCredentials([usernamePassword(credentialsId: 'GITHUB_INTEGRATION', usernameVariable: 'GH_AUTH_USERNAME', passwordVariable: 'GH_AUTH_PASSWORD')]) { - sh "curl $GIT_API_URL \ - --data '{ \ - \"state\" : \"success\", \ - \"target_url\": \"$TARGET_URL\", \ - \"description\": \"Your build succeeded!\", \ - \"context\": \"ci/jenkins\" }' \ - -u $GH_AUTH_USERNAME:$GH_AUTH_PASSWORD" - } - } - - failure { - mail(from: "${EMAIL_ALERT_FROM}", - to: "${EMAIL_ALERT_TO}", - subject: "Jenkins build failed: ${JOB_NAME}:${BUILD_NUMBER}", - body: "Build: ${BUILD_URL}") - withCredentials([usernamePassword(credentialsId: 'GITHUB_INTEGRATION', usernameVariable: 'GH_AUTH_USERNAME', passwordVariable: 'GH_AUTH_PASSWORD')]) { - sh "curl $GIT_API_URL \ - --data '{ \ - \"state\" : \"failure\", \ - \"target_url\": \"$TARGET_URL\", \ - \"description\": \"Your build failed\", \ - \"context\": \"ci/jenkins\" }' \ - -u $GH_AUTH_USERNAME:$GH_AUTH_PASSWORD" - } - } - } - - // The options directive is for configuration that applies to the whole job. - options { - // we'd like to make sure remove old builds, so we don't fill up our storage! - buildDiscarder(logRotator(numToKeepStr:'50')) - - // And we'd really like to be sure that this build doesn't hang forever, so let's time it out after: - timeout(time: 30, unit: 'MINUTES') - } -} diff --git a/services/real-time/Makefile b/services/real-time/Makefile index 437700ee2f..a569868e5b 100644 --- a/services/real-time/Makefile +++ b/services/real-time/Makefile @@ -25,13 +25,13 @@ clean: docker rmi gcr.io/overleaf-ops/$(PROJECT_NAME):$(BRANCH_NAME)-$(BUILD_NUMBER) format: - $(DOCKER_COMPOSE) run --rm test_unit npm run format + $(DOCKER_COMPOSE) run --rm test_unit npm run --silent format format_fix: - $(DOCKER_COMPOSE) run --rm test_unit npm run format:fix + $(DOCKER_COMPOSE) run --rm test_unit npm run --silent format:fix lint: - $(DOCKER_COMPOSE) run --rm test_unit npm run lint + $(DOCKER_COMPOSE) run --rm test_unit npm run --silent lint test: format lint test_unit test_acceptance diff --git a/services/real-time/buildscript.txt b/services/real-time/buildscript.txt index 526bd01e14..742bdcd841 100644 --- a/services/real-time/buildscript.txt +++ b/services/real-time/buildscript.txt @@ -1,10 +1,8 @@ real-time ---acceptance-creds=None --dependencies=redis --docker-repos=gcr.io/overleaf-ops --env-add= --env-pass-through= ---language=es --node-version=10.21.0 --public-repo=True ---script-version=2.3.0 +--script-version=3.3.0 diff --git a/services/real-time/docker-compose.ci.yml b/services/real-time/docker-compose.ci.yml index f9fc7b983e..255a6b77a6 100644 --- a/services/real-time/docker-compose.ci.yml +++ b/services/real-time/docker-compose.ci.yml @@ -11,6 +11,7 @@ services: command: npm run test:unit:_run environment: NODE_ENV: test + NODE_OPTIONS: "--unhandled-rejections=strict" test_acceptance: @@ -23,6 +24,7 @@ services: POSTGRES_HOST: postgres MOCHA_GREP: ${MOCHA_GREP} NODE_ENV: test + NODE_OPTIONS: "--unhandled-rejections=strict" depends_on: redis: condition: service_healthy diff --git a/services/real-time/docker-compose.yml b/services/real-time/docker-compose.yml index 8570f506ae..4525e7d83d 100644 --- a/services/real-time/docker-compose.yml +++ b/services/real-time/docker-compose.yml @@ -13,7 +13,8 @@ services: environment: MOCHA_GREP: ${MOCHA_GREP} NODE_ENV: test - command: npm run test:unit + NODE_OPTIONS: "--unhandled-rejections=strict" + command: npm run --silent test:unit user: node test_acceptance: @@ -29,11 +30,12 @@ services: MOCHA_GREP: ${MOCHA_GREP} LOG_LEVEL: ERROR NODE_ENV: test + NODE_OPTIONS: "--unhandled-rejections=strict" user: node depends_on: redis: condition: service_healthy - command: npm run test:acceptance + command: npm run --silent test:acceptance redis: image: redis diff --git a/services/real-time/nodemon.json b/services/real-time/nodemon.json index 5826281b84..e3e8817d90 100644 --- a/services/real-time/nodemon.json +++ b/services/real-time/nodemon.json @@ -8,7 +8,6 @@ "execMap": { "js": "npm run start" }, - "watch": [ "app/js/", "app.js", diff --git a/services/real-time/package.json b/services/real-time/package.json index be1ebc79bb..d94a39554b 100644 --- a/services/real-time/package.json +++ b/services/real-time/package.json @@ -14,7 +14,7 @@ "test:unit:_run": "mocha --recursive --reporter spec $@ test/unit/js", "test:unit": "npm run test:unit:_run -- --grep=$MOCHA_GREP", "nodemon": "nodemon --config nodemon.json", - "lint": "node_modules/.bin/eslint .", + "lint": "node_modules/.bin/eslint --max-warnings 0 .", "format": "node_modules/.bin/prettier-eslint $PWD'/**/*.js' --list-different", "format:fix": "node_modules/.bin/prettier-eslint $PWD'/**/*.js' --write" }, From 51e56dafcf36e055b79dc3a126caf49c6b6d6a35 Mon Sep 17 00:00:00 2001 From: Jakob Ackermann Date: Wed, 5 Aug 2020 10:46:23 +0100 Subject: [PATCH 403/491] [misc] bump the dev-env to 3.3.1 --- services/real-time/.gitignore | 3 +++ services/real-time/.npmrc | 2 -- services/real-time/buildscript.txt | 2 +- 3 files changed, 4 insertions(+), 3 deletions(-) delete mode 100644 services/real-time/.npmrc diff --git a/services/real-time/.gitignore b/services/real-time/.gitignore index 50678c09e9..80bac793a7 100644 --- a/services/real-time/.gitignore +++ b/services/real-time/.gitignore @@ -1,2 +1,5 @@ node_modules forever + +# managed by dev-environment$ bin/update_build_scripts +.npmrc diff --git a/services/real-time/.npmrc b/services/real-time/.npmrc deleted file mode 100644 index 34ecfacbc4..0000000000 --- a/services/real-time/.npmrc +++ /dev/null @@ -1,2 +0,0 @@ -# block local, outside docker invocations of $ npm install -dry-run=true diff --git a/services/real-time/buildscript.txt b/services/real-time/buildscript.txt index 742bdcd841..c4e9ac7c8d 100644 --- a/services/real-time/buildscript.txt +++ b/services/real-time/buildscript.txt @@ -5,4 +5,4 @@ real-time --env-pass-through= --node-version=10.21.0 --public-repo=True ---script-version=3.3.0 +--script-version=3.3.1 From 370baa06c330e6e27356917dc6975c8daa1e637a Mon Sep 17 00:00:00 2001 From: Jakob Ackermann Date: Mon, 10 Aug 2020 17:10:31 +0100 Subject: [PATCH 404/491] [misc] bump the dev-env to 3.3.2 --- services/real-time/buildscript.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/services/real-time/buildscript.txt b/services/real-time/buildscript.txt index c4e9ac7c8d..96813e9029 100644 --- a/services/real-time/buildscript.txt +++ b/services/real-time/buildscript.txt @@ -5,4 +5,4 @@ real-time --env-pass-through= --node-version=10.21.0 --public-repo=True ---script-version=3.3.1 +--script-version=3.3.2 From 831d794bf46a9dbfaf97adca13ff234649f02f7c Mon Sep 17 00:00:00 2001 From: Brian Gough Date: Mon, 13 Jul 2020 10:42:50 +0100 Subject: [PATCH 405/491] clean up join/leave handling Co-Authored-By: Jakob Ackermann --- .../app/js/DocumentUpdaterManager.js | 9 +- services/real-time/app/js/Router.js | 3 + .../real-time/app/js/WebsocketController.js | 50 +++++- .../test/unit/js/WebsocketControllerTests.js | 144 +++++++++++++++++- .../test/unit/js/helpers/MockClient.js | 1 + 5 files changed, 204 insertions(+), 3 deletions(-) diff --git a/services/real-time/app/js/DocumentUpdaterManager.js b/services/real-time/app/js/DocumentUpdaterManager.js index 32533a9eaa..3fd09d7b1d 100644 --- a/services/real-time/app/js/DocumentUpdaterManager.js +++ b/services/real-time/app/js/DocumentUpdaterManager.js @@ -12,7 +12,7 @@ const rclient = require('redis-sharelatex').createClient( ) const Keys = settings.redis.documentupdater.key_schema -module.exports = { +const DocumentUpdaterManager = { getDocument(project_id, doc_id, fromVersion, callback) { const timer = new metrics.Timer('get-document') const url = `${settings.apis.documentupdater.url}/project/${project_id}/doc/${doc_id}?fromVersion=${fromVersion}` @@ -63,6 +63,11 @@ module.exports = { }) }, + checkDocument(project_id, doc_id, callback) { + // in this call fromVersion = -1 means get document without docOps + DocumentUpdaterManager.getDocument(project_id, doc_id, -1, callback) + }, + flushProjectToMongoAndDelete(project_id, callback) { // this method is called when the last connected user leaves the project logger.log({ project_id }, 'deleting project from document updater') @@ -141,3 +146,5 @@ module.exports = { }) } } + +module.exports = DocumentUpdaterManager diff --git a/services/real-time/app/js/Router.js b/services/real-time/app/js/Router.js index 0b4b3f00d7..ae10d12e2b 100644 --- a/services/real-time/app/js/Router.js +++ b/services/real-time/app/js/Router.js @@ -45,6 +45,7 @@ module.exports = Router = { } else if ( [ 'not authorized', + 'joinLeaveEpoch mismatch', 'doc updater could not load requested ops', 'no project_id found on client' ].includes(error.message) @@ -95,6 +96,8 @@ module.exports = Router = { // init client context, we may access it in Router._handleError before // setting any values client.ol_context = {} + // bail out from joinDoc when a parallel joinDoc or leaveDoc is running + client.joinLeaveEpoch = 0 if (client) { client.on('error', function (err) { diff --git a/services/real-time/app/js/WebsocketController.js b/services/real-time/app/js/WebsocketController.js index 239372eea9..8867320dbe 100644 --- a/services/real-time/app/js/WebsocketController.js +++ b/services/real-time/app/js/WebsocketController.js @@ -158,6 +158,7 @@ module.exports = WebsocketController = { return callback() } + const joinLeaveEpoch = ++client.joinLeaveEpoch metrics.inc('editor.join-doc') const { project_id, user_id, is_restricted_user } = client.ol_context if (!project_id) { @@ -168,10 +169,23 @@ module.exports = WebsocketController = { 'client joining doc' ) - AuthorizationManager.assertClientCanViewProject(client, function (error) { + WebsocketController._assertClientAuthorization(client, doc_id, function ( + error + ) { if (error) { return callback(error) } + if (client.disconnected) { + metrics.inc('editor.join-doc.disconnected', 1, { + status: 'after-client-auth-check' + }) + // the client will not read the response anyways + return callback() + } + if (joinLeaveEpoch !== client.joinLeaveEpoch) { + // another joinDoc or leaveDoc rpc overtook us + return callback(new Error('joinLeaveEpoch mismatch')) + } // ensure the per-doc applied-ops channel is subscribed before sending the // doc to the client, so that no events are missed. RoomManager.joinDoc(client, doc_id, function (error) { @@ -279,8 +293,42 @@ module.exports = WebsocketController = { }) }, + _assertClientAuthorization(client, doc_id, callback) { + // Check for project-level access first + AuthorizationManager.assertClientCanViewProject(client, function (error) { + if (error) { + return callback(error) + } + // Check for doc-level access next + AuthorizationManager.assertClientCanViewProjectAndDoc( + client, + doc_id, + function (error) { + if (error) { + // No cached access, check docupdater + const { project_id } = client.ol_context + DocumentUpdaterManager.checkDocument(project_id, doc_id, function ( + error + ) { + if (error) { + return callback(error) + } else { + // Success + AuthorizationManager.addAccessToDoc(client, doc_id, callback) + } + }) + } else { + // Access already cached + callback() + } + } + ) + }) + }, + leaveDoc(client, doc_id, callback) { // client may have disconnected, but we have to cleanup internal state. + client.joinLeaveEpoch++ metrics.inc('editor.leave-doc') const { project_id, user_id } = client.ol_context logger.log( diff --git a/services/real-time/test/unit/js/WebsocketControllerTests.js b/services/real-time/test/unit/js/WebsocketControllerTests.js index 7796ca7275..515d407cc2 100644 --- a/services/real-time/test/unit/js/WebsocketControllerTests.js +++ b/services/real-time/test/unit/js/WebsocketControllerTests.js @@ -38,6 +38,7 @@ describe('WebsocketController', function () { id: (this.client_id = 'mock-client-id-123'), publicId: `other-id-${Math.random()}`, ol_context: {}, + joinLeaveEpoch: 0, join: sinon.stub(), leave: sinon.stub() } @@ -503,10 +504,13 @@ describe('WebsocketController', function () { this.client.ol_context.project_id = this.project_id this.client.ol_context.is_restricted_user = false - this.AuthorizationManager.addAccessToDoc = sinon.stub() + this.AuthorizationManager.addAccessToDoc = sinon.stub().yields() this.AuthorizationManager.assertClientCanViewProject = sinon .stub() .callsArgWith(1, null) + this.AuthorizationManager.assertClientCanViewProjectAndDoc = sinon + .stub() + .callsArgWith(2, null) this.DocumentUpdaterManager.getDocument = sinon .stub() .callsArgWith( @@ -531,6 +535,10 @@ describe('WebsocketController', function () { ) }) + it('should inc the joinLeaveEpoch', function () { + expect(this.client.joinLeaveEpoch).to.equal(1) + }) + it('should check that the client is authorized to view the project', function () { return this.AuthorizationManager.assertClientCanViewProject .calledWith(this.client) @@ -739,6 +747,136 @@ describe('WebsocketController', function () { }) }) + describe('when the client disconnects while auth checks are running', function () { + beforeEach(function (done) { + this.AuthorizationManager.assertClientCanViewProjectAndDoc.yields( + new Error() + ) + this.DocumentUpdaterManager.checkDocument = ( + project_id, + doc_id, + cb + ) => { + this.client.disconnected = true + cb() + } + + this.WebsocketController.joinDoc( + this.client, + this.doc_id, + -1, + this.options, + (...args) => { + this.callback(...args) + done() + } + ) + }) + + it('should call the callback with no details', function () { + expect(this.callback.called).to.equal(true) + expect(this.callback.args[0]).to.deep.equal([]) + }) + + it('should increment the editor.join-doc.disconnected metric with a status', function () { + expect( + this.metrics.inc.calledWith('editor.join-doc.disconnected', 1, { + status: 'after-client-auth-check' + }) + ).to.equal(true) + }) + + it('should not get the document', function () { + expect(this.DocumentUpdaterManager.getDocument.called).to.equal(false) + }) + }) + + describe('when the client starts a parallel joinDoc request', function () { + beforeEach(function (done) { + this.AuthorizationManager.assertClientCanViewProjectAndDoc.yields( + new Error() + ) + this.DocumentUpdaterManager.checkDocument = ( + project_id, + doc_id, + cb + ) => { + this.DocumentUpdaterManager.checkDocument = sinon.stub().yields() + this.WebsocketController.joinDoc( + this.client, + this.doc_id, + -1, + {}, + () => {} + ) + cb() + } + + this.WebsocketController.joinDoc( + this.client, + this.doc_id, + -1, + this.options, + (...args) => { + this.callback(...args) + // make sure the other joinDoc request completed + setTimeout(done, 5) + } + ) + }) + + it('should call the callback with an error', function () { + expect(this.callback.called).to.equal(true) + expect(this.callback.args[0][0].message).to.equal( + 'joinLeaveEpoch mismatch' + ) + }) + + it('should get the document once (the parallel request wins)', function () { + expect(this.DocumentUpdaterManager.getDocument.callCount).to.equal(1) + }) + }) + + describe('when the client starts a parallel leaveDoc request', function () { + beforeEach(function (done) { + this.RoomManager.leaveDoc = sinon.stub() + + this.AuthorizationManager.assertClientCanViewProjectAndDoc.yields( + new Error() + ) + this.DocumentUpdaterManager.checkDocument = ( + project_id, + doc_id, + cb + ) => { + this.WebsocketController.leaveDoc(this.client, this.doc_id, () => {}) + cb() + } + + this.WebsocketController.joinDoc( + this.client, + this.doc_id, + -1, + this.options, + (...args) => { + this.callback(...args) + done() + } + ) + }) + + it('should call the callback with an error', function () { + expect(this.callback.called).to.equal(true) + expect(this.callback.args[0][0].message).to.equal( + 'joinLeaveEpoch mismatch' + ) + }) + + it('should not get the document', function () { + expect(this.DocumentUpdaterManager.getDocument.called).to.equal(false) + }) + }) + describe('when the client disconnects while RoomManager.joinDoc is running', function () { beforeEach(function () { this.RoomManager.joinDoc = (client, doc_id, cb) => { @@ -827,6 +965,10 @@ describe('WebsocketController', function () { ) }) + it('should inc the joinLeaveEpoch', function () { + expect(this.client.joinLeaveEpoch).to.equal(1) + }) + it('should remove the client from the doc_id room', function () { return this.RoomManager.leaveDoc .calledWith(this.client, this.doc_id) diff --git a/services/real-time/test/unit/js/helpers/MockClient.js b/services/real-time/test/unit/js/helpers/MockClient.js index fb90a0f7e9..61cde89ba9 100644 --- a/services/real-time/test/unit/js/helpers/MockClient.js +++ b/services/real-time/test/unit/js/helpers/MockClient.js @@ -16,6 +16,7 @@ module.exports = MockClient = class MockClient { this.disconnect = sinon.stub() this.id = idCounter++ this.publicId = idCounter++ + this.joinLeaveEpoch = 0 } disconnect() {} From a0028646a5a9140c25c4abfcfc4d3d54f2992cb5 Mon Sep 17 00:00:00 2001 From: Jakob Ackermann Date: Wed, 12 Aug 2020 11:45:36 +0100 Subject: [PATCH 406/491] [app] ignore errors from accessing disconnected client sockets Technically `EHOSTUNREACH` and `ETIMEDOUT` should go into a 'disconnected_read' metric. But this would require changes to dashboards and a larger diff. --- services/real-time/app.js | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/services/real-time/app.js b/services/real-time/app.js index c41743e597..fa6c5f7563 100644 --- a/services/real-time/app.js +++ b/services/real-time/app.js @@ -181,7 +181,11 @@ if (Settings.shutdownDrainTimeWindow) { if (Settings.errors && Settings.errors.catchUncaughtErrors) { process.removeAllListeners('uncaughtException') process.on('uncaughtException', function (error) { - if (['EPIPE', 'ECONNRESET'].includes(error.code)) { + if ( + ['ETIMEDOUT', 'EHOSTUNREACH', 'EPIPE', 'ECONNRESET'].includes( + error.code + ) + ) { Metrics.inc('disconnected_write', 1, { status: error.code }) return logger.warn( { err: error }, From fe41f1b00cfe1bd16192948dda2d6127919b46e8 Mon Sep 17 00:00:00 2001 From: Christopher Hoskin Date: Wed, 12 Aug 2020 16:04:21 +0100 Subject: [PATCH 407/491] [misc] bump logger-sharelatex to version 2.2.0 --- services/real-time/package-lock.json | 934 ++++++++++++++++++++++++--- services/real-time/package.json | 2 +- 2 files changed, 828 insertions(+), 108 deletions(-) diff --git a/services/real-time/package-lock.json b/services/real-time/package-lock.json index 045283d72a..a56634e4c2 100644 --- a/services/real-time/package-lock.json +++ b/services/real-time/package-lock.json @@ -76,6 +76,353 @@ } } }, + "@google-cloud/logging": { + "version": "7.3.0", + "resolved": "https://registry.npmjs.org/@google-cloud/logging/-/logging-7.3.0.tgz", + "integrity": "sha512-xTW1V4MKpYC0mjSugyuiyUoZ9g6A42IhrrO3z7Tt3SmAb2IRj2Gf4RLoguKKncs340ooZFXrrVN/++t2Aj5zgg==", + "requires": { + "@google-cloud/common": "^2.2.2", + "@google-cloud/paginator": "^2.0.0", + "@google-cloud/projectify": "^1.0.0", + "@google-cloud/promisify": "^1.0.0", + "@opencensus/propagation-stackdriver": "0.0.20", + "arrify": "^2.0.0", + "dot-prop": "^5.1.0", + "eventid": "^1.0.0", + "extend": "^3.0.2", + "gcp-metadata": "^3.1.0", + "google-auth-library": "^5.2.2", + "google-gax": "^1.11.0", + "is": "^3.3.0", + "on-finished": "^2.3.0", + "pumpify": "^2.0.0", + "snakecase-keys": "^3.0.0", + "stream-events": "^1.0.4", + "through2": "^3.0.0", + "type-fest": "^0.12.0" + }, + "dependencies": { + "@google-cloud/common": { + "version": "2.4.0", + "resolved": "https://registry.npmjs.org/@google-cloud/common/-/common-2.4.0.tgz", + "integrity": "sha512-zWFjBS35eI9leAHhjfeOYlK5Plcuj/77EzstnrJIZbKgF/nkqjcQuGiMCpzCwOfPyUbz8ZaEOYgbHa759AKbjg==", + "requires": { + "@google-cloud/projectify": "^1.0.0", + "@google-cloud/promisify": "^1.0.0", + "arrify": "^2.0.0", + "duplexify": "^3.6.0", + "ent": "^2.2.0", + "extend": "^3.0.2", + "google-auth-library": "^5.5.0", + "retry-request": "^4.0.0", + "teeny-request": "^6.0.0" + } + }, + "@google-cloud/projectify": { + "version": "1.0.4", + "resolved": "https://registry.npmjs.org/@google-cloud/projectify/-/projectify-1.0.4.tgz", + "integrity": "sha512-ZdzQUN02eRsmTKfBj9FDL0KNDIFNjBn/d6tHQmA/+FImH5DO6ZV8E7FzxMgAUiVAUq41RFAkb25p1oHOZ8psfg==" + }, + "@google-cloud/promisify": { + "version": "1.0.4", + "resolved": "https://registry.npmjs.org/@google-cloud/promisify/-/promisify-1.0.4.tgz", + "integrity": "sha512-VccZDcOql77obTnFh0TbNED/6ZbbmHDf8UMNnzO1d5g9V0Htfm4k5cllY8P1tJsRKC3zWYGRLaViiupcgVjBoQ==" + }, + "agent-base": { + "version": "6.0.1", + "resolved": "https://registry.npmjs.org/agent-base/-/agent-base-6.0.1.tgz", + "integrity": "sha512-01q25QQDwLSsyfhrKbn8yuur+JNw0H+0Y4JiGIKd3z9aYk/w/2kxD/Upc+t2ZBBSUNff50VjPsSW2YxM8QYKVg==", + "requires": { + "debug": "4" + } + }, + "debug": { + "version": "4.1.1", + "resolved": "https://registry.npmjs.org/debug/-/debug-4.1.1.tgz", + "integrity": "sha512-pYAIzeRo8J6KPEaJ0VWOh5Pzkbw/RetuzehGM7QRRX5he4fPHx2rdKMB256ehJCkX+XRQm16eZLqLNS8RSZXZw==", + "requires": { + "ms": "^2.1.1" + } + }, + "gaxios": { + "version": "2.3.4", + "resolved": "https://registry.npmjs.org/gaxios/-/gaxios-2.3.4.tgz", + "integrity": "sha512-US8UMj8C5pRnao3Zykc4AAVr+cffoNKRTg9Rsf2GiuZCW69vgJj38VK2PzlPuQU73FZ/nTk9/Av6/JGcE1N9vA==", + "requires": { + "abort-controller": "^3.0.0", + "extend": "^3.0.2", + "https-proxy-agent": "^5.0.0", + "is-stream": "^2.0.0", + "node-fetch": "^2.3.0" + } + }, + "gcp-metadata": { + "version": "3.5.0", + "resolved": "https://registry.npmjs.org/gcp-metadata/-/gcp-metadata-3.5.0.tgz", + "integrity": "sha512-ZQf+DLZ5aKcRpLzYUyBS3yo3N0JSa82lNDO8rj3nMSlovLcz2riKFBsYgDzeXcv75oo5eqB2lx+B14UvPoCRnA==", + "requires": { + "gaxios": "^2.1.0", + "json-bigint": "^0.3.0" + } + }, + "google-auth-library": { + "version": "5.10.1", + "resolved": "https://registry.npmjs.org/google-auth-library/-/google-auth-library-5.10.1.tgz", + "integrity": "sha512-rOlaok5vlpV9rSiUu5EpR0vVpc+PhN62oF4RyX/6++DG1VsaulAFEMlDYBLjJDDPI6OcNOCGAKy9UVB/3NIDXg==", + "requires": { + "arrify": "^2.0.0", + "base64-js": "^1.3.0", + "ecdsa-sig-formatter": "^1.0.11", + "fast-text-encoding": "^1.0.0", + "gaxios": "^2.1.0", + "gcp-metadata": "^3.4.0", + "gtoken": "^4.1.0", + "jws": "^4.0.0", + "lru-cache": "^5.0.0" + } + }, + "google-p12-pem": { + "version": "2.0.4", + "resolved": "https://registry.npmjs.org/google-p12-pem/-/google-p12-pem-2.0.4.tgz", + "integrity": "sha512-S4blHBQWZRnEW44OcR7TL9WR+QCqByRvhNDZ/uuQfpxywfupikf/miba8js1jZi6ZOGv5slgSuoshCWh6EMDzg==", + "requires": { + "node-forge": "^0.9.0" + } + }, + "gtoken": { + "version": "4.1.4", + "resolved": "https://registry.npmjs.org/gtoken/-/gtoken-4.1.4.tgz", + "integrity": "sha512-VxirzD0SWoFUo5p8RDP8Jt2AGyOmyYcT/pOUgDKJCK+iSw0TMqwrVfY37RXTNmoKwrzmDHSk0GMT9FsgVmnVSA==", + "requires": { + "gaxios": "^2.1.0", + "google-p12-pem": "^2.0.0", + "jws": "^4.0.0", + "mime": "^2.2.0" + } + }, + "https-proxy-agent": { + "version": "5.0.0", + "resolved": "https://registry.npmjs.org/https-proxy-agent/-/https-proxy-agent-5.0.0.tgz", + "integrity": "sha512-EkYm5BcKUGiduxzSt3Eppko+PiNWNEpa4ySk9vTC6wDsQJW9rHSa+UhGNJoRYp7bz6Ht1eaRIa6QaJqO5rCFbA==", + "requires": { + "agent-base": "6", + "debug": "4" + } + }, + "jwa": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/jwa/-/jwa-2.0.0.tgz", + "integrity": "sha512-jrZ2Qx916EA+fq9cEAeCROWPTfCwi1IVHqT2tapuqLEVVDKFDENFw1oL+MwrTvH6msKxsd1YTDVw6uKEcsrLEA==", + "requires": { + "buffer-equal-constant-time": "1.0.1", + "ecdsa-sig-formatter": "1.0.11", + "safe-buffer": "^5.0.1" + } + }, + "jws": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/jws/-/jws-4.0.0.tgz", + "integrity": "sha512-KDncfTmOZoOMTFG4mBlG0qUIOlc03fmzH+ru6RgYVZhPkyiy/92Owlt/8UEN+a4TXR1FQetfIpJE8ApdvdVxTg==", + "requires": { + "jwa": "^2.0.0", + "safe-buffer": "^5.0.1" + } + }, + "mime": { + "version": "2.4.6", + "resolved": "https://registry.npmjs.org/mime/-/mime-2.4.6.tgz", + "integrity": "sha512-RZKhC3EmpBchfTGBVb8fb+RL2cWyw/32lshnsETttkBAyAUXSGHxbEJWWRXc751DrIxG1q04b8QwMbAwkRPpUA==" + }, + "ms": { + "version": "2.1.2", + "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.2.tgz", + "integrity": "sha512-sGkPx+VjMtmA6MX27oA4FBFELFCZZ4S4XqeGOXCv68tT+jb3vk/RyaKWP0PTKyWtmLSM0b+adUTEvbs1PEaH2w==" + }, + "node-forge": { + "version": "0.9.1", + "resolved": "https://registry.npmjs.org/node-forge/-/node-forge-0.9.1.tgz", + "integrity": "sha512-G6RlQt5Sb4GMBzXvhfkeFmbqR6MzhtnT7VTHuLadjkii3rdYHNdw0m8zA4BTxVIh68FicCQ2NSUANpsqkr9jvQ==" + }, + "teeny-request": { + "version": "6.0.3", + "resolved": "https://registry.npmjs.org/teeny-request/-/teeny-request-6.0.3.tgz", + "integrity": "sha512-TZG/dfd2r6yeji19es1cUIwAlVD8y+/svB1kAC2Y0bjEyysrfbO8EZvJBRwIE6WkwmUoB7uvWLwTIhJbMXZ1Dw==", + "requires": { + "http-proxy-agent": "^4.0.0", + "https-proxy-agent": "^5.0.0", + "node-fetch": "^2.2.0", + "stream-events": "^1.0.5", + "uuid": "^7.0.0" + } + }, + "type-fest": { + "version": "0.12.0", + "resolved": "https://registry.npmjs.org/type-fest/-/type-fest-0.12.0.tgz", + "integrity": "sha512-53RyidyjvkGpnWPMF9bQgFtWp+Sl8O2Rp13VavmJgfAP9WWG6q6TkrKU8iyJdnwnfgHI6k2hTlgqH4aSdjoTbg==" + } + } + }, + "@google-cloud/logging-bunyan": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/@google-cloud/logging-bunyan/-/logging-bunyan-3.0.0.tgz", + "integrity": "sha512-ZLVXEejNQ27ktGcA3S/sd7GPefp7kywbn+/KoBajdb1Syqcmtc98jhXpYQBXVtNP2065iyu77s4SBaiYFbTC5A==", + "requires": { + "@google-cloud/logging": "^7.0.0", + "google-auth-library": "^6.0.0" + }, + "dependencies": { + "agent-base": { + "version": "6.0.1", + "resolved": "https://registry.npmjs.org/agent-base/-/agent-base-6.0.1.tgz", + "integrity": "sha512-01q25QQDwLSsyfhrKbn8yuur+JNw0H+0Y4JiGIKd3z9aYk/w/2kxD/Upc+t2ZBBSUNff50VjPsSW2YxM8QYKVg==", + "requires": { + "debug": "4" + } + }, + "bignumber.js": { + "version": "9.0.0", + "resolved": "https://registry.npmjs.org/bignumber.js/-/bignumber.js-9.0.0.tgz", + "integrity": "sha512-t/OYhhJ2SD+YGBQcjY8GzzDHEk9f3nerxjtfa6tlMXfe7frs/WozhvCNoGvpM0P3bNf3Gq5ZRMlGr5f3r4/N8A==" + }, + "debug": { + "version": "4.1.1", + "resolved": "https://registry.npmjs.org/debug/-/debug-4.1.1.tgz", + "integrity": "sha512-pYAIzeRo8J6KPEaJ0VWOh5Pzkbw/RetuzehGM7QRRX5he4fPHx2rdKMB256ehJCkX+XRQm16eZLqLNS8RSZXZw==", + "requires": { + "ms": "^2.1.1" + } + }, + "gaxios": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/gaxios/-/gaxios-3.1.0.tgz", + "integrity": "sha512-DDTn3KXVJJigtz+g0J3vhcfbDbKtAroSTxauWsdnP57sM5KZ3d2c/3D9RKFJ86s43hfw6WULg6TXYw/AYiBlpA==", + "requires": { + "abort-controller": "^3.0.0", + "extend": "^3.0.2", + "https-proxy-agent": "^5.0.0", + "is-stream": "^2.0.0", + "node-fetch": "^2.3.0" + } + }, + "gcp-metadata": { + "version": "4.1.4", + "resolved": "https://registry.npmjs.org/gcp-metadata/-/gcp-metadata-4.1.4.tgz", + "integrity": "sha512-5J/GIH0yWt/56R3dNaNWPGQ/zXsZOddYECfJaqxFWgrZ9HC2Kvc5vl9upOgUUHKzURjAVf2N+f6tEJiojqXUuA==", + "requires": { + "gaxios": "^3.0.0", + "json-bigint": "^1.0.0" + } + }, + "google-auth-library": { + "version": "6.0.6", + "resolved": "https://registry.npmjs.org/google-auth-library/-/google-auth-library-6.0.6.tgz", + "integrity": "sha512-fWYdRdg55HSJoRq9k568jJA1lrhg9i2xgfhVIMJbskUmbDpJGHsbv9l41DGhCDXM21F9Kn4kUwdysgxSYBYJUw==", + "requires": { + "arrify": "^2.0.0", + "base64-js": "^1.3.0", + "ecdsa-sig-formatter": "^1.0.11", + "fast-text-encoding": "^1.0.0", + "gaxios": "^3.0.0", + "gcp-metadata": "^4.1.0", + "gtoken": "^5.0.0", + "jws": "^4.0.0", + "lru-cache": "^6.0.0" + } + }, + "google-p12-pem": { + "version": "3.0.2", + "resolved": "https://registry.npmjs.org/google-p12-pem/-/google-p12-pem-3.0.2.tgz", + "integrity": "sha512-tbjzndQvSIHGBLzHnhDs3cL4RBjLbLXc2pYvGH+imGVu5b4RMAttUTdnmW2UH0t11QeBTXZ7wlXPS7hrypO/tg==", + "requires": { + "node-forge": "^0.9.0" + } + }, + "gtoken": { + "version": "5.0.3", + "resolved": "https://registry.npmjs.org/gtoken/-/gtoken-5.0.3.tgz", + "integrity": "sha512-Nyd1wZCMRc2dj/mAD0LlfQLcAO06uKdpKJXvK85SGrF5+5+Bpfil9u/2aw35ltvEHjvl0h5FMKN5knEU+9JrOg==", + "requires": { + "gaxios": "^3.0.0", + "google-p12-pem": "^3.0.0", + "jws": "^4.0.0", + "mime": "^2.2.0" + } + }, + "https-proxy-agent": { + "version": "5.0.0", + "resolved": "https://registry.npmjs.org/https-proxy-agent/-/https-proxy-agent-5.0.0.tgz", + "integrity": "sha512-EkYm5BcKUGiduxzSt3Eppko+PiNWNEpa4ySk9vTC6wDsQJW9rHSa+UhGNJoRYp7bz6Ht1eaRIa6QaJqO5rCFbA==", + "requires": { + "agent-base": "6", + "debug": "4" + } + }, + "json-bigint": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/json-bigint/-/json-bigint-1.0.0.tgz", + "integrity": "sha512-SiPv/8VpZuWbvLSMtTDU8hEfrZWg/mH/nV/b4o0CYbSxu1UIQPLdwKOCIyLQX+VIPO5vrLX3i8qtqFyhdPSUSQ==", + "requires": { + "bignumber.js": "^9.0.0" + } + }, + "jwa": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/jwa/-/jwa-2.0.0.tgz", + "integrity": "sha512-jrZ2Qx916EA+fq9cEAeCROWPTfCwi1IVHqT2tapuqLEVVDKFDENFw1oL+MwrTvH6msKxsd1YTDVw6uKEcsrLEA==", + "requires": { + "buffer-equal-constant-time": "1.0.1", + "ecdsa-sig-formatter": "1.0.11", + "safe-buffer": "^5.0.1" + } + }, + "jws": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/jws/-/jws-4.0.0.tgz", + "integrity": "sha512-KDncfTmOZoOMTFG4mBlG0qUIOlc03fmzH+ru6RgYVZhPkyiy/92Owlt/8UEN+a4TXR1FQetfIpJE8ApdvdVxTg==", + "requires": { + "jwa": "^2.0.0", + "safe-buffer": "^5.0.1" + } + }, + "lru-cache": { + "version": "6.0.0", + "resolved": "https://registry.npmjs.org/lru-cache/-/lru-cache-6.0.0.tgz", + "integrity": "sha512-Jo6dJ04CmSjuznwJSS3pUeWmd/H0ffTlkXXgwZi+eq1UCmqQwCh+eLsYOYCwY991i2Fah4h1BEMCx4qThGbsiA==", + "requires": { + "yallist": "^4.0.0" + } + }, + "mime": { + "version": "2.4.6", + "resolved": "https://registry.npmjs.org/mime/-/mime-2.4.6.tgz", + "integrity": "sha512-RZKhC3EmpBchfTGBVb8fb+RL2cWyw/32lshnsETttkBAyAUXSGHxbEJWWRXc751DrIxG1q04b8QwMbAwkRPpUA==" + }, + "ms": { + "version": "2.1.2", + "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.2.tgz", + "integrity": "sha512-sGkPx+VjMtmA6MX27oA4FBFELFCZZ4S4XqeGOXCv68tT+jb3vk/RyaKWP0PTKyWtmLSM0b+adUTEvbs1PEaH2w==" + }, + "node-forge": { + "version": "0.9.1", + "resolved": "https://registry.npmjs.org/node-forge/-/node-forge-0.9.1.tgz", + "integrity": "sha512-G6RlQt5Sb4GMBzXvhfkeFmbqR6MzhtnT7VTHuLadjkii3rdYHNdw0m8zA4BTxVIh68FicCQ2NSUANpsqkr9jvQ==" + }, + "yallist": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/yallist/-/yallist-4.0.0.tgz", + "integrity": "sha512-3wdGidZyq5PB084XLES5TpOSRA3wjXAlIWMhum2kRcv/41Sn2emQ0dycQW4uZXLejwKvg6EsvbdlVL+FYEct7A==" + } + } + }, + "@google-cloud/paginator": { + "version": "2.0.3", + "resolved": "https://registry.npmjs.org/@google-cloud/paginator/-/paginator-2.0.3.tgz", + "integrity": "sha512-kp/pkb2p/p0d8/SKUu4mOq8+HGwF8NPzHWkj+VKrIPQPyMRw8deZtrO/OcSiy9C/7bpfU5Txah5ltUNfPkgEXg==", + "requires": { + "arrify": "^2.0.0", + "extend": "^3.0.2" + } + }, "@google-cloud/profiler": { "version": "0.2.3", "resolved": "https://registry.npmjs.org/@google-cloud/profiler/-/profiler-0.2.3.tgz", @@ -206,6 +553,64 @@ } } }, + "@grpc/grpc-js": { + "version": "1.0.5", + "resolved": "https://registry.npmjs.org/@grpc/grpc-js/-/grpc-js-1.0.5.tgz", + "integrity": "sha512-Hm+xOiqAhcpT9RYM8lc15dbQD7aQurM7ZU8ulmulepiPlN7iwBXXwP3vSBUimoFoApRqz7pSIisXU8pZaCB4og==", + "requires": { + "semver": "^6.2.0" + } + }, + "@grpc/proto-loader": { + "version": "0.5.5", + "resolved": "https://registry.npmjs.org/@grpc/proto-loader/-/proto-loader-0.5.5.tgz", + "integrity": "sha512-WwN9jVNdHRQoOBo9FDH7qU+mgfjPc8GygPYms3M+y3fbQLfnCe/Kv/E01t7JRgnrsOHH8euvSbed3mIalXhwqQ==", + "requires": { + "lodash.camelcase": "^4.3.0", + "protobufjs": "^6.8.6" + } + }, + "@opencensus/core": { + "version": "0.0.20", + "resolved": "https://registry.npmjs.org/@opencensus/core/-/core-0.0.20.tgz", + "integrity": "sha512-vqOuTd2yuMpKohp8TNNGUAPjWEGjlnGfB9Rh5e3DKqeyR94YgierNs4LbMqxKtsnwB8Dm2yoEtRuUgoe5vD9DA==", + "requires": { + "continuation-local-storage": "^3.2.1", + "log-driver": "^1.2.7", + "semver": "^6.0.0", + "shimmer": "^1.2.0", + "uuid": "^3.2.1" + }, + "dependencies": { + "uuid": { + "version": "3.4.0", + "resolved": "https://registry.npmjs.org/uuid/-/uuid-3.4.0.tgz", + "integrity": "sha512-HjSDRw6gZE5JMggctHBcjVak08+KEVhSIiDzFnT9S9aegmp85S/bReBVTb4QTFaRNptJ9kuYaNhnbNEOkbKb/A==" + } + } + }, + "@opencensus/propagation-stackdriver": { + "version": "0.0.20", + "resolved": "https://registry.npmjs.org/@opencensus/propagation-stackdriver/-/propagation-stackdriver-0.0.20.tgz", + "integrity": "sha512-P8yuHSLtce+yb+2EZjtTVqG7DQ48laC+IuOWi3X9q78s1Gni5F9+hmbmyP6Nb61jb5BEvXQX1s2rtRI6bayUWA==", + "requires": { + "@opencensus/core": "^0.0.20", + "hex2dec": "^1.0.1", + "uuid": "^3.2.1" + }, + "dependencies": { + "uuid": { + "version": "3.4.0", + "resolved": "https://registry.npmjs.org/uuid/-/uuid-3.4.0.tgz", + "integrity": "sha512-HjSDRw6gZE5JMggctHBcjVak08+KEVhSIiDzFnT9S9aegmp85S/bReBVTb4QTFaRNptJ9kuYaNhnbNEOkbKb/A==" + } + } + }, + "@overleaf/o-error": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/@overleaf/o-error/-/o-error-3.0.0.tgz", + "integrity": "sha512-LsM2s6Iy9G97ktPo0ys4VxtI/m3ahc1ZHwjo5XnhXtjeIkkkVAehsrcRRoV/yWepPjymB0oZonhcfojpjYR/tg==" + }, "@protobufjs/aspromise": { "version": "1.1.2", "resolved": "https://registry.npmjs.org/@protobufjs/aspromise/-/aspromise-1.1.2.tgz", @@ -265,6 +670,11 @@ "resolved": "https://registry.npmjs.org/@sindresorhus/is/-/is-0.15.0.tgz", "integrity": "sha512-lu8BpxjAtRCAo5ifytTpCPCj99LF7o/2Myn+NXyNCBqvPYn7Pjd76AMmUB5l7XF1U6t0hcWrlEM5ESufW7wAeA==" }, + "@tootallnate/once": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/@tootallnate/once/-/once-1.1.2.tgz", + "integrity": "sha512-RbzJvlNzmRq5c3O09UipeuXno4tA1FE6ikOjxZK0tuxVv3412l64l5t1W5pj4+rJq9vpkm/kwiR07aZXnsKPxw==" + }, "@types/caseless": { "version": "0.12.2", "resolved": "https://registry.npmjs.org/@types/caseless/-/caseless-0.12.2.tgz", @@ -295,6 +705,14 @@ "integrity": "sha512-OCutwjDZ4aFS6PB1UZ988C4YgwlBHJd6wCeQqaLdmadZ/7e+w79+hbMUFC1QXDNCmdyoRfAFdm0RypzwR+Qpag==", "dev": true }, + "@types/fs-extra": { + "version": "8.1.1", + "resolved": "https://registry.npmjs.org/@types/fs-extra/-/fs-extra-8.1.1.tgz", + "integrity": "sha512-TcUlBem321DFQzBNuz8p0CLLKp0VvF/XH9E4KHNmgwyp4E3AfgI5cjiIVZWlbfThBop2qxFIh4+LeY6hVWWZ2w==", + "requires": { + "@types/node": "*" + } + }, "@types/json-schema": { "version": "7.0.5", "resolved": "https://registry.npmjs.org/@types/json-schema/-/json-schema-7.0.5.tgz", @@ -779,6 +1197,11 @@ "integrity": "sha512-mT8iDcrh03qDGRRmoA2hmBJnxpllMR+0/0qlzjqZES6NdiWDcZkCNAk4rPFZ9Q85r27unkiNNg8ZOiwZXBHwcA==", "dev": true }, + "charenc": { + "version": "0.0.2", + "resolved": "https://registry.npmjs.org/charenc/-/charenc-0.0.2.tgz", + "integrity": "sha1-wKHS86cJLgN3S/qD8UwPxXkKhmc=" + }, "cli-cursor": { "version": "3.1.0", "resolved": "https://registry.npmjs.org/cli-cursor/-/cli-cursor-3.1.0.tgz", @@ -941,7 +1364,7 @@ "cookie": { "version": "0.3.1", "resolved": "https://registry.npmjs.org/cookie/-/cookie-0.3.1.tgz", - "integrity": "sha512-+IJOX0OqlHCszo2mBUq+SrEbCj6w7Kpffqx60zYbPTFaO4+yYgRjHwcZNpWvaTylDHaV7PPmBHzSecZiMhtPgw==" + "integrity": "sha1-5+Ch+e9DtMi6klxcWpboBtFoc7s=" }, "cookie-parser": { "version": "1.4.5", @@ -1002,6 +1425,16 @@ } } }, + "crypt": { + "version": "0.0.2", + "resolved": "https://registry.npmjs.org/crypt/-/crypt-0.0.2.tgz", + "integrity": "sha1-iNf/fsDfuG9xPch7u0LQRNPmxBs=" + }, + "d64": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/d64/-/d64-1.0.0.tgz", + "integrity": "sha1-QAKofoUMv8n52XBrYPymE6MzbpA=" + }, "dashdash": { "version": "1.14.1", "resolved": "https://registry.npmjs.org/dashdash/-/dashdash-1.14.1.tgz", @@ -1094,6 +1527,14 @@ "esutils": "^2.0.2" } }, + "dot-prop": { + "version": "5.2.0", + "resolved": "https://registry.npmjs.org/dot-prop/-/dot-prop-5.2.0.tgz", + "integrity": "sha512-uEUyaDKoSQ1M4Oq8l45hSE26SnTxL6snNnqvK/VWx5wJhmff5z0FUVJDKDanor/6w3kzE3i7XZOk+7wC0EXr1A==", + "requires": { + "is-obj": "^2.0.0" + } + }, "dtrace-provider": { "version": "0.2.8", "resolved": "https://registry.npmjs.org/dtrace-provider/-/dtrace-provider-0.2.8.tgz", @@ -1578,6 +2019,22 @@ "resolved": "https://registry.npmjs.org/event-target-shim/-/event-target-shim-5.0.1.tgz", "integrity": "sha512-i/2XbnSz/uxRCU6+NdVJgKWDTM427+MqYbkQzD321DuCQJUqOuJKIA0IM2+W2xtYHdKOmZ4dR6fExsd4SXL+WQ==" }, + "eventid": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/eventid/-/eventid-1.0.0.tgz", + "integrity": "sha512-4upSDsvpxhWPsmw4fsJCp0zj8S7I0qh1lCDTmZXP8V3TtryQKDI8CgQPN+e5JakbWwzaAX3lrdp2b3KSoMSUpw==", + "requires": { + "d64": "^1.0.0", + "uuid": "^3.0.1" + }, + "dependencies": { + "uuid": { + "version": "3.4.0", + "resolved": "https://registry.npmjs.org/uuid/-/uuid-3.4.0.tgz", + "integrity": "sha512-HjSDRw6gZE5JMggctHBcjVak08+KEVhSIiDzFnT9S9aegmp85S/bReBVTb4QTFaRNptJ9kuYaNhnbNEOkbKb/A==" + } + } + }, "express": { "version": "4.17.1", "resolved": "https://registry.npmjs.org/express/-/express-4.17.1.tgz", @@ -1974,6 +2431,145 @@ } } }, + "google-gax": { + "version": "1.15.3", + "resolved": "https://registry.npmjs.org/google-gax/-/google-gax-1.15.3.tgz", + "integrity": "sha512-3JKJCRumNm3x2EksUTw4P1Rad43FTpqrtW9jzpf3xSMYXx+ogaqTM1vGo7VixHB4xkAyATXVIa3OcNSh8H9zsQ==", + "requires": { + "@grpc/grpc-js": "~1.0.3", + "@grpc/proto-loader": "^0.5.1", + "@types/fs-extra": "^8.0.1", + "@types/long": "^4.0.0", + "abort-controller": "^3.0.0", + "duplexify": "^3.6.0", + "google-auth-library": "^5.0.0", + "is-stream-ended": "^0.1.4", + "lodash.at": "^4.6.0", + "lodash.has": "^4.5.2", + "node-fetch": "^2.6.0", + "protobufjs": "^6.8.9", + "retry-request": "^4.0.0", + "semver": "^6.0.0", + "walkdir": "^0.4.0" + }, + "dependencies": { + "agent-base": { + "version": "6.0.1", + "resolved": "https://registry.npmjs.org/agent-base/-/agent-base-6.0.1.tgz", + "integrity": "sha512-01q25QQDwLSsyfhrKbn8yuur+JNw0H+0Y4JiGIKd3z9aYk/w/2kxD/Upc+t2ZBBSUNff50VjPsSW2YxM8QYKVg==", + "requires": { + "debug": "4" + } + }, + "debug": { + "version": "4.1.1", + "resolved": "https://registry.npmjs.org/debug/-/debug-4.1.1.tgz", + "integrity": "sha512-pYAIzeRo8J6KPEaJ0VWOh5Pzkbw/RetuzehGM7QRRX5he4fPHx2rdKMB256ehJCkX+XRQm16eZLqLNS8RSZXZw==", + "requires": { + "ms": "^2.1.1" + } + }, + "gaxios": { + "version": "2.3.4", + "resolved": "https://registry.npmjs.org/gaxios/-/gaxios-2.3.4.tgz", + "integrity": "sha512-US8UMj8C5pRnao3Zykc4AAVr+cffoNKRTg9Rsf2GiuZCW69vgJj38VK2PzlPuQU73FZ/nTk9/Av6/JGcE1N9vA==", + "requires": { + "abort-controller": "^3.0.0", + "extend": "^3.0.2", + "https-proxy-agent": "^5.0.0", + "is-stream": "^2.0.0", + "node-fetch": "^2.3.0" + } + }, + "gcp-metadata": { + "version": "3.5.0", + "resolved": "https://registry.npmjs.org/gcp-metadata/-/gcp-metadata-3.5.0.tgz", + "integrity": "sha512-ZQf+DLZ5aKcRpLzYUyBS3yo3N0JSa82lNDO8rj3nMSlovLcz2riKFBsYgDzeXcv75oo5eqB2lx+B14UvPoCRnA==", + "requires": { + "gaxios": "^2.1.0", + "json-bigint": "^0.3.0" + } + }, + "google-auth-library": { + "version": "5.10.1", + "resolved": "https://registry.npmjs.org/google-auth-library/-/google-auth-library-5.10.1.tgz", + "integrity": "sha512-rOlaok5vlpV9rSiUu5EpR0vVpc+PhN62oF4RyX/6++DG1VsaulAFEMlDYBLjJDDPI6OcNOCGAKy9UVB/3NIDXg==", + "requires": { + "arrify": "^2.0.0", + "base64-js": "^1.3.0", + "ecdsa-sig-formatter": "^1.0.11", + "fast-text-encoding": "^1.0.0", + "gaxios": "^2.1.0", + "gcp-metadata": "^3.4.0", + "gtoken": "^4.1.0", + "jws": "^4.0.0", + "lru-cache": "^5.0.0" + } + }, + "google-p12-pem": { + "version": "2.0.4", + "resolved": "https://registry.npmjs.org/google-p12-pem/-/google-p12-pem-2.0.4.tgz", + "integrity": "sha512-S4blHBQWZRnEW44OcR7TL9WR+QCqByRvhNDZ/uuQfpxywfupikf/miba8js1jZi6ZOGv5slgSuoshCWh6EMDzg==", + "requires": { + "node-forge": "^0.9.0" + } + }, + "gtoken": { + "version": "4.1.4", + "resolved": "https://registry.npmjs.org/gtoken/-/gtoken-4.1.4.tgz", + "integrity": "sha512-VxirzD0SWoFUo5p8RDP8Jt2AGyOmyYcT/pOUgDKJCK+iSw0TMqwrVfY37RXTNmoKwrzmDHSk0GMT9FsgVmnVSA==", + "requires": { + "gaxios": "^2.1.0", + "google-p12-pem": "^2.0.0", + "jws": "^4.0.0", + "mime": "^2.2.0" + } + }, + "https-proxy-agent": { + "version": "5.0.0", + "resolved": "https://registry.npmjs.org/https-proxy-agent/-/https-proxy-agent-5.0.0.tgz", + "integrity": "sha512-EkYm5BcKUGiduxzSt3Eppko+PiNWNEpa4ySk9vTC6wDsQJW9rHSa+UhGNJoRYp7bz6Ht1eaRIa6QaJqO5rCFbA==", + "requires": { + "agent-base": "6", + "debug": "4" + } + }, + "jwa": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/jwa/-/jwa-2.0.0.tgz", + "integrity": "sha512-jrZ2Qx916EA+fq9cEAeCROWPTfCwi1IVHqT2tapuqLEVVDKFDENFw1oL+MwrTvH6msKxsd1YTDVw6uKEcsrLEA==", + "requires": { + "buffer-equal-constant-time": "1.0.1", + "ecdsa-sig-formatter": "1.0.11", + "safe-buffer": "^5.0.1" + } + }, + "jws": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/jws/-/jws-4.0.0.tgz", + "integrity": "sha512-KDncfTmOZoOMTFG4mBlG0qUIOlc03fmzH+ru6RgYVZhPkyiy/92Owlt/8UEN+a4TXR1FQetfIpJE8ApdvdVxTg==", + "requires": { + "jwa": "^2.0.0", + "safe-buffer": "^5.0.1" + } + }, + "mime": { + "version": "2.4.6", + "resolved": "https://registry.npmjs.org/mime/-/mime-2.4.6.tgz", + "integrity": "sha512-RZKhC3EmpBchfTGBVb8fb+RL2cWyw/32lshnsETttkBAyAUXSGHxbEJWWRXc751DrIxG1q04b8QwMbAwkRPpUA==" + }, + "ms": { + "version": "2.1.2", + "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.2.tgz", + "integrity": "sha512-sGkPx+VjMtmA6MX27oA4FBFELFCZZ4S4XqeGOXCv68tT+jb3vk/RyaKWP0PTKyWtmLSM0b+adUTEvbs1PEaH2w==" + }, + "node-forge": { + "version": "0.9.1", + "resolved": "https://registry.npmjs.org/node-forge/-/node-forge-0.9.1.tgz", + "integrity": "sha512-G6RlQt5Sb4GMBzXvhfkeFmbqR6MzhtnT7VTHuLadjkii3rdYHNdw0m8zA4BTxVIh68FicCQ2NSUANpsqkr9jvQ==" + } + } + }, "google-p12-pem": { "version": "1.0.4", "resolved": "https://registry.npmjs.org/google-p12-pem/-/google-p12-pem-1.0.4.tgz", @@ -2089,6 +2685,39 @@ "toidentifier": "1.0.0" } }, + "http-proxy-agent": { + "version": "4.0.1", + "resolved": "https://registry.npmjs.org/http-proxy-agent/-/http-proxy-agent-4.0.1.tgz", + "integrity": "sha512-k0zdNgqWTGA6aeIRVpvfVob4fL52dTfaehylg0Y4UvSySvOq/Y+BOyPrgpUrA7HylqvU8vIZGsRuXmspskV0Tg==", + "requires": { + "@tootallnate/once": "1", + "agent-base": "6", + "debug": "4" + }, + "dependencies": { + "agent-base": { + "version": "6.0.1", + "resolved": "https://registry.npmjs.org/agent-base/-/agent-base-6.0.1.tgz", + "integrity": "sha512-01q25QQDwLSsyfhrKbn8yuur+JNw0H+0Y4JiGIKd3z9aYk/w/2kxD/Upc+t2ZBBSUNff50VjPsSW2YxM8QYKVg==", + "requires": { + "debug": "4" + } + }, + "debug": { + "version": "4.1.1", + "resolved": "https://registry.npmjs.org/debug/-/debug-4.1.1.tgz", + "integrity": "sha512-pYAIzeRo8J6KPEaJ0VWOh5Pzkbw/RetuzehGM7QRRX5he4fPHx2rdKMB256ehJCkX+XRQm16eZLqLNS8RSZXZw==", + "requires": { + "ms": "^2.1.1" + } + }, + "ms": { + "version": "2.1.2", + "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.2.tgz", + "integrity": "sha512-sGkPx+VjMtmA6MX27oA4FBFELFCZZ4S4XqeGOXCv68tT+jb3vk/RyaKWP0PTKyWtmLSM0b+adUTEvbs1PEaH2w==" + } + } + }, "http-signature": { "version": "1.2.0", "resolved": "https://registry.npmjs.org/http-signature/-/http-signature-1.2.0.tgz", @@ -2340,6 +2969,11 @@ "is-extglob": "^2.1.1" } }, + "is-obj": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/is-obj/-/is-obj-2.0.0.tgz", + "integrity": "sha512-drqDG3cbczxxEJRoOXcOjtdp1J/lyp1mNn0xaznRs8+muBhgQcrnbspox5X5fOw0HnMnbfDzvnEMEtqDEJEo8w==" + }, "is-regex": { "version": "1.1.0", "resolved": "https://registry.npmjs.org/is-regex/-/is-regex-1.1.0.tgz", @@ -2349,6 +2983,16 @@ "has-symbols": "^1.0.1" } }, + "is-stream": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/is-stream/-/is-stream-2.0.0.tgz", + "integrity": "sha512-XCoy+WlUr7d1+Z8GgSuXmpuUFC9fOhRXglJMx+dwLKTkL44Cjd4W1Z5P+BQZpr+cR93aGP4S/s7Ftw6Nd/kiEw==" + }, + "is-stream-ended": { + "version": "0.1.4", + "resolved": "https://registry.npmjs.org/is-stream-ended/-/is-stream-ended-0.1.4.tgz", + "integrity": "sha512-xj0XPvmr7bQFTvirqnFr50o0hQIh6ZItDqloxt5aJrR4NQsYeSsyFQERYGCAzfindAcnKjINnwEEgLx4IqVzQw==" + }, "is-string": { "version": "1.0.5", "resolved": "https://registry.npmjs.org/is-string/-/is-string-1.0.5.tgz", @@ -2527,6 +3171,16 @@ "resolved": "https://registry.npmjs.org/lodash/-/lodash-4.17.15.tgz", "integrity": "sha512-8xOcRHvCjnocdS5cpwXQXVzmmh5e5+saE2QGoeQmbKmRS6J3VQppPOIt0MnmE+4xlZoumy0GPG0D0MVIQbNA1A==" }, + "lodash.at": { + "version": "4.6.0", + "resolved": "https://registry.npmjs.org/lodash.at/-/lodash.at-4.6.0.tgz", + "integrity": "sha1-k83OZk8KGZTqM9181A4jr9EbD/g=" + }, + "lodash.camelcase": { + "version": "4.3.0", + "resolved": "https://registry.npmjs.org/lodash.camelcase/-/lodash.camelcase-4.3.0.tgz", + "integrity": "sha1-soqmKIorn8ZRA1x3EfZathkDMaY=" + }, "lodash.defaults": { "version": "4.2.0", "resolved": "https://registry.npmjs.org/lodash.defaults/-/lodash.defaults-4.2.0.tgz", @@ -2537,6 +3191,11 @@ "resolved": "https://registry.npmjs.org/lodash.flatten/-/lodash.flatten-4.4.0.tgz", "integrity": "sha512-C5N2Z3DgnnKr0LOpv/hKCgKdb7ZZwafIrsesve6lmzvZIRZRGaZ/l6Q8+2W7NaT+ZwO3fFlSCzCzrDCFdJfZ4g==" }, + "lodash.has": { + "version": "4.5.2", + "resolved": "https://registry.npmjs.org/lodash.has/-/lodash.has-4.5.2.tgz", + "integrity": "sha1-0Z9NwQlQWMzL4rDN9O4P5Ko3yGI=" + }, "lodash.memoize": { "version": "4.1.2", "resolved": "https://registry.npmjs.org/lodash.memoize/-/lodash.memoize-4.1.2.tgz", @@ -2560,90 +3219,54 @@ "integrity": "sha512-DhhGRshNS1aX6s5YdBE3njCCouPgnG29ebyHvImlZzXZf2SHgt+J08DHgytTPnpywNbO1Y8mNUFyQuIDBq2JZg==", "dev": true }, + "log-driver": { + "version": "1.2.7", + "resolved": "https://registry.npmjs.org/log-driver/-/log-driver-1.2.7.tgz", + "integrity": "sha512-U7KCmLdqsGHBLeWqYlFA0V0Sl6P08EE1ZrmA9cxjUE0WVqT9qnyVDPz1kzpFEP0jdJuFnasWIfSd7fsaNXkpbg==" + }, "logger-sharelatex": { - "version": "1.7.0", - "resolved": "https://registry.npmjs.org/logger-sharelatex/-/logger-sharelatex-1.7.0.tgz", - "integrity": "sha512-9sxDGPSphOMDqUqGpOu/KxFAVcpydKggWv60g9D7++FDCxGkhLLn0kmBkDdgB00d1PadgX1CBMWKzIBpptDU/Q==", + "version": "2.2.0", + "resolved": "https://registry.npmjs.org/logger-sharelatex/-/logger-sharelatex-2.2.0.tgz", + "integrity": "sha512-ko+OmE25XHJJCiz1R9EgwlfM7J/5olpunUfR3WcfuqOQrcUqsdBrDA2sOytngT0ViwjCR0Fh4qZVPwEWfmrvwA==", "requires": { - "bunyan": "1.8.12", - "raven": "1.1.3", - "request": "2.88.0" + "@google-cloud/logging-bunyan": "^3.0.0", + "@overleaf/o-error": "^3.0.0", + "bunyan": "^1.8.14", + "node-fetch": "^2.6.0", + "raven": "^2.6.4", + "yn": "^4.0.0" }, "dependencies": { "bunyan": { - "version": "1.8.12", - "resolved": "https://registry.npmjs.org/bunyan/-/bunyan-1.8.12.tgz", - "integrity": "sha512-dmDUbGHeGcvCDLRFOscZkwx1ZO/aFz3bJOCi5nCgzdhFGPxwK+y5AcDBnqagNGlJZ7lje/l6JUEz9mQcutttdg==", + "version": "1.8.14", + "resolved": "https://registry.npmjs.org/bunyan/-/bunyan-1.8.14.tgz", + "integrity": "sha512-LlahJUxXzZLuw/hetUQJmRgZ1LF6+cr5TPpRj6jf327AsiIq2jhYEH4oqUUkVKTor+9w2BT3oxVwhzE5lw9tcg==", "requires": { "dtrace-provider": "~0.8", - "moment": "^2.10.6", + "moment": "^2.19.3", "mv": "~2", "safe-json-stringify": "~1" } }, "dtrace-provider": { - "version": "0.8.7", - "resolved": "https://registry.npmjs.org/dtrace-provider/-/dtrace-provider-0.8.7.tgz", - "integrity": "sha512-V+HIGbAdxCIxddHNDwzXi6cx8Cz5RRlQOVcsryHfsyVVebpBEnDwHSgqxpgKzqeU/6/0DWqRLAGUwkbg2ecN1Q==", + "version": "0.8.8", + "resolved": "https://registry.npmjs.org/dtrace-provider/-/dtrace-provider-0.8.8.tgz", + "integrity": "sha512-b7Z7cNtHPhH9EJhNNbbeqTcXB8LGFFZhq1PGgEvpeHlzd36bhbdTWoE/Ba/YguqpBSlAPKnARWhVlhunCMwfxg==", "optional": true, "requires": { - "nan": "^2.10.0" + "nan": "^2.14.0" } }, - "mime-db": { - "version": "1.44.0", - "resolved": "https://registry.npmjs.org/mime-db/-/mime-db-1.44.0.tgz", - "integrity": "sha512-/NOTfLrsPBVeH7YtFPgsVWveuL+4SjjYxaQ1xtM1KMFj7HdxlBlxeyNLzhyJVx7r4rZGJAZ/6lkKCitSc/Nmpg==" + "nan": { + "version": "2.14.1", + "resolved": "https://registry.npmjs.org/nan/-/nan-2.14.1.tgz", + "integrity": "sha512-isWHgVjnFjh2x2yuJ/tj3JbwoHu3UC2dX5G/88Cm24yB6YopVgxvBObDY7n5xW6ExmFhJpSEQqFPvq9zaXc8Jw==", + "optional": true }, - "mime-types": { - "version": "2.1.27", - "resolved": "https://registry.npmjs.org/mime-types/-/mime-types-2.1.27.tgz", - "integrity": "sha512-JIhqnCasI9yD+SsmkquHBxTSEuZdQX5BuQnS2Vc7puQQQ+8yiP5AY5uWhpdv4YL4VM5c6iliiYWPgJ/nJQLp7w==", - "requires": { - "mime-db": "1.44.0" - } - }, - "qs": { - "version": "6.5.2", - "resolved": "https://registry.npmjs.org/qs/-/qs-6.5.2.tgz", - "integrity": "sha512-N5ZAX4/LxJmF+7wN74pUD6qAh9/wnvdQcjq9TZjevvXzSUo7bfmw91saqMjzGS2xq91/odN2dW/WOl7qQHNDGA==" - }, - "request": { - "version": "2.88.0", - "resolved": "https://registry.npmjs.org/request/-/request-2.88.0.tgz", - "integrity": "sha512-NAqBSrijGLZdM0WZNsInLJpkJokL72XYjUpnB0iwsRgxh7dB6COrHnTBNwN0E+lHDAJzu7kLAkDeY08z2/A0hg==", - "requires": { - "aws-sign2": "~0.7.0", - "aws4": "^1.8.0", - "caseless": "~0.12.0", - "combined-stream": "~1.0.6", - "extend": "~3.0.2", - "forever-agent": "~0.6.1", - "form-data": "~2.3.2", - "har-validator": "~5.1.0", - "http-signature": "~1.2.0", - "is-typedarray": "~1.0.0", - "isstream": "~0.1.2", - "json-stringify-safe": "~5.0.1", - "mime-types": "~2.1.19", - "oauth-sign": "~0.9.0", - "performance-now": "^2.1.0", - "qs": "~6.5.2", - "safe-buffer": "^5.1.2", - "tough-cookie": "~2.4.3", - "tunnel-agent": "^0.6.0", - "uuid": "^3.3.2" - } - }, - "safe-buffer": { - "version": "5.2.1", - "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.2.1.tgz", - "integrity": "sha512-rp3So07KcdmmKbGvgaNxQSJr7bGVSVk5S9Eq1F+ppbRo70+YeaDxkw5Dd8NPN+GD6bjnYm2VuPuCXmpuYvmCXQ==" - }, - "uuid": { - "version": "3.4.0", - "resolved": "https://registry.npmjs.org/uuid/-/uuid-3.4.0.tgz", - "integrity": "sha512-HjSDRw6gZE5JMggctHBcjVak08+KEVhSIiDzFnT9S9aegmp85S/bReBVTb4QTFaRNptJ9kuYaNhnbNEOkbKb/A==" + "yn": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/yn/-/yn-4.0.0.tgz", + "integrity": "sha512-huWiiCS4TxKc4SfgmTwW1K7JmXPPAmuXWYy4j9qjQo4+27Kni8mGhAAi1cloRWmBe2EqcLgt3IGqQoRL/MtPgg==" } } }, @@ -2724,11 +3347,6 @@ "yallist": "^3.0.2" } }, - "lsmod": { - "version": "1.0.0", - "resolved": "https://registry.npmjs.org/lsmod/-/lsmod-1.0.0.tgz", - "integrity": "sha512-Y+6V75r+mGWzWEPr9h6PFmStielICu5JBHLUg18jCsD2VFmEfgHbq/EgnY4inElsUD9eKL9id1qp34w46rSIKQ==" - }, "lynx": { "version": "0.1.1", "resolved": "https://registry.npmjs.org/lynx/-/lynx-0.1.1.tgz", @@ -2759,8 +3377,24 @@ "map-obj": { "version": "4.1.0", "resolved": "https://registry.npmjs.org/map-obj/-/map-obj-4.1.0.tgz", - "integrity": "sha512-glc9y00wgtwcDmp7GaE/0b0OnxpNJsVf3ael/An6Fe2Q51LLwN1er6sdomLRzz5h0+yMpiYLhWYF5R7HeqVd4g==", - "dev": true + "integrity": "sha512-glc9y00wgtwcDmp7GaE/0b0OnxpNJsVf3ael/An6Fe2Q51LLwN1er6sdomLRzz5h0+yMpiYLhWYF5R7HeqVd4g==" + }, + "md5": { + "version": "2.3.0", + "resolved": "https://registry.npmjs.org/md5/-/md5-2.3.0.tgz", + "integrity": "sha512-T1GITYmFaKuO91vxyoQMFETst+O71VUPEU3ze5GNzDm0OWdP8v1ziTaAEPUr/3kLsY3Sftgz242A1SetQiDL7g==", + "requires": { + "charenc": "0.0.2", + "crypt": "0.0.2", + "is-buffer": "~1.1.6" + }, + "dependencies": { + "is-buffer": { + "version": "1.1.6", + "resolved": "https://registry.npmjs.org/is-buffer/-/is-buffer-1.1.6.tgz", + "integrity": "sha512-NcdALwpXkTm5Zvvbk7owOUSvVvBKDgKP5/ewfXEznmQFfs4ZRmanOeKBTjRVjka3QFoN6XJ+9F3USqfHqTaU5w==" + } + } }, "media-typer": { "version": "0.3.0", @@ -2954,9 +3588,9 @@ "integrity": "sha512-ySViT69/76t8VhE1xXHK6Ch4NcDd26gx0MzKXLO+F7NOtnqH68d9zF94nT8ZWSxXh8ELOERsnJO/sWt1xZYw5A==" }, "moment": { - "version": "2.24.0", - "resolved": "https://registry.npmjs.org/moment/-/moment-2.24.0.tgz", - "integrity": "sha512-bV7f+6l2QigeBBZSM/6yTNq4P2fNpSWj/0e7jQcy87A8e7o2nAfP/34/2ky5Vw4B9S446EtIhodAzkFCcR4dQg==", + "version": "2.27.0", + "resolved": "https://registry.npmjs.org/moment/-/moment-2.27.0.tgz", + "integrity": "sha512-al0MUK7cpIcglMv3YF13qSgdAIqxHTO7brRtaz3DlSULbqfazqkc5kEjNrLDOM7fsjshoFIihnU8snrP7zUvhQ==", "optional": true }, "ms": { @@ -3976,6 +4610,48 @@ "resolved": "https://registry.npmjs.org/psl/-/psl-1.8.0.tgz", "integrity": "sha512-RIdOzyoavK+hA18OGGWDqUTsCLhtA7IcZ/6NCs4fFJaHBDab+pDDmDIByWFRQJq2Cd7r1OoQxBGKOaztq+hjIQ==" }, + "pump": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/pump/-/pump-3.0.0.tgz", + "integrity": "sha512-LwZy+p3SFs1Pytd/jYct4wpv49HiYCqd9Rlc5ZVdk0V+8Yzv6jR5Blk3TRmPL1ft69TxP0IMZGJ+WPFU2BFhww==", + "requires": { + "end-of-stream": "^1.1.0", + "once": "^1.3.1" + } + }, + "pumpify": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/pumpify/-/pumpify-2.0.1.tgz", + "integrity": "sha512-m7KOje7jZxrmutanlkS1daj1dS6z6BgslzOXmcSEpIlCxM3VJH7lG5QLeck/6hgF6F4crFf01UtQmNsJfweTAw==", + "requires": { + "duplexify": "^4.1.1", + "inherits": "^2.0.3", + "pump": "^3.0.0" + }, + "dependencies": { + "duplexify": { + "version": "4.1.1", + "resolved": "https://registry.npmjs.org/duplexify/-/duplexify-4.1.1.tgz", + "integrity": "sha512-DY3xVEmVHTv1wSzKNbwoU6nVjzI369Y6sPoqfYr0/xlx3IdX2n94xIszTcjPO8W8ZIv0Wb0PXNcjuZyT4wiICA==", + "requires": { + "end-of-stream": "^1.4.1", + "inherits": "^2.0.3", + "readable-stream": "^3.1.1", + "stream-shift": "^1.0.0" + } + }, + "readable-stream": { + "version": "3.6.0", + "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-3.6.0.tgz", + "integrity": "sha512-BViHy7LKeTz4oNnkcLJ+lVSL6vpiFeX6/d3oSH8zCW7UxP2onchk+vTGB143xuFjHS3deTgkKoXXymXqymiIdA==", + "requires": { + "inherits": "^2.0.3", + "string_decoder": "^1.1.1", + "util-deprecate": "^1.0.1" + } + } + } + }, "punycode": { "version": "2.1.1", "resolved": "https://registry.npmjs.org/punycode/-/punycode-2.1.1.tgz", @@ -4014,15 +4690,22 @@ "integrity": "sha512-Hrgsx+orqoygnmhFbKaHE6c296J+HTAQXoxEF6gNupROmmGJRoyzfG3ccAveqCBrwr/2yxQ5BVd/GTl5agOwSg==" }, "raven": { - "version": "1.1.3", - "resolved": "https://registry.npmjs.org/raven/-/raven-1.1.3.tgz", - "integrity": "sha512-RYov4wAaflZasWiCrZuizd3jNXxCOkW1WrXgWsGVb8kRpdHNZ+vPY27R6RhVtqzWp+DG9a5l6iP0QUPK4EgzaQ==", + "version": "2.6.4", + "resolved": "https://registry.npmjs.org/raven/-/raven-2.6.4.tgz", + "integrity": "sha512-6PQdfC4+DQSFncowthLf+B6Hr0JpPsFBgTVYTAOq7tCmx/kR4SXbeawtPch20+3QfUcQDoJBLjWW1ybvZ4kXTw==", "requires": { "cookie": "0.3.1", - "json-stringify-safe": "5.0.1", - "lsmod": "1.0.0", - "stack-trace": "0.0.9", - "uuid": "3.0.0" + "md5": "^2.2.1", + "stack-trace": "0.0.10", + "timed-out": "4.0.1", + "uuid": "3.3.2" + }, + "dependencies": { + "uuid": { + "version": "3.3.2", + "resolved": "https://registry.npmjs.org/uuid/-/uuid-3.3.2.tgz", + "integrity": "sha512-yXJmeNaw3DnnKAOKJE51sL/ZaYfWJRl1pK9dr19YFCu0ObS231AB1/LbqTKRAQ5kw8A90rA6fr4riOUpTZvQZA==" + } } }, "raw-body": { @@ -4536,6 +5219,15 @@ } } }, + "snakecase-keys": { + "version": "3.2.0", + "resolved": "https://registry.npmjs.org/snakecase-keys/-/snakecase-keys-3.2.0.tgz", + "integrity": "sha512-WTJ0NhCH/37J+PU3fuz0x5b6TvtWQChTcKPOndWoUy0pteKOe0hrHMzSRsJOWSIP48EQkzUEsgQPmrG3W8pFNQ==", + "requires": { + "map-obj": "^4.0.0", + "to-snake-case": "^1.0.0" + } + }, "socket.io": { "version": "https://github.com/overleaf/socket.io/archive/0.9.19-overleaf-3.tar.gz", "integrity": "sha512-x07f3i4f3/CXaTBpUT/mH4JtYf3xjZWO0tvIB43Nn8OnzeRhMaKxhMZWcLQxfPdcaAyvafwLeM06fiDpHCaIYw==", @@ -4632,9 +5324,9 @@ } }, "stack-trace": { - "version": "0.0.9", - "resolved": "https://registry.npmjs.org/stack-trace/-/stack-trace-0.0.9.tgz", - "integrity": "sha512-vjUc6sfgtgY0dxCdnc40mK6Oftjo9+2K8H/NG81TMhgL392FtiPA9tn9RLyTxXmTLPJPjF3VyzFp6bsWFLisMQ==" + "version": "0.0.10", + "resolved": "https://registry.npmjs.org/stack-trace/-/stack-trace-0.0.10.tgz", + "integrity": "sha1-VHxws0fo0ytOEI6hoqFZ5f3eGcA=" }, "standard-as-callback": { "version": "2.0.1", @@ -4651,6 +5343,14 @@ "resolved": "https://registry.npmjs.org/statuses/-/statuses-1.5.0.tgz", "integrity": "sha512-OpZ3zP+jT1PI7I8nemJX4AKmAX070ZkYPVWV/AaKTJl+tXCTGyVdC1a4SL8RUQYEwk/f34ZX8UTykN68FwrqAA==" }, + "stream-events": { + "version": "1.0.5", + "resolved": "https://registry.npmjs.org/stream-events/-/stream-events-1.0.5.tgz", + "integrity": "sha512-E1GUzBSgvct8Jsb3v2X15pjzN1tYebtbLaMg+eBOUOAxgbLoSbT2NS91ckc5lJD1KfLjId+jXJRgo0qnV5Nerg==", + "requires": { + "stubs": "^3.0.0" + } + }, "stream-shift": { "version": "1.0.1", "resolved": "https://registry.npmjs.org/stream-shift/-/stream-shift-1.0.1.tgz", @@ -4735,6 +5435,11 @@ "integrity": "sha512-e6/d0eBu7gHtdCqFt0xJr642LdToM5/cN4Qb9DbHjVx1CP5RyeM+zH7pbecEmDv/lBqb0QH+6Uqq75rxFPkM0w==", "dev": true }, + "stubs": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/stubs/-/stubs-3.0.0.tgz", + "integrity": "sha1-6NK6H6nJBXAwPAMLaQD31fiavls=" + }, "supports-color": { "version": "5.5.0", "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-5.5.0.tgz", @@ -4831,6 +5536,11 @@ "readable-stream": "2 || 3" } }, + "timed-out": { + "version": "4.0.1", + "resolved": "https://registry.npmjs.org/timed-out/-/timed-out-4.0.1.tgz", + "integrity": "sha1-8y6srFoXW+ol1/q1Zas+2HQe9W8=" + }, "timekeeper": { "version": "0.0.4", "resolved": "https://registry.npmjs.org/timekeeper/-/timekeeper-0.0.4.tgz", @@ -4851,27 +5561,32 @@ "os-tmpdir": "~1.0.2" } }, + "to-no-case": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/to-no-case/-/to-no-case-1.0.2.tgz", + "integrity": "sha1-xyKQcWTvaxeBMsjmmTAhLRtKoWo=" + }, + "to-snake-case": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/to-snake-case/-/to-snake-case-1.0.0.tgz", + "integrity": "sha1-znRpE4l5RgGah+Yu366upMYIq4w=", + "requires": { + "to-space-case": "^1.0.0" + } + }, + "to-space-case": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/to-space-case/-/to-space-case-1.0.0.tgz", + "integrity": "sha1-sFLar7Gysp3HcM6gFj5ewOvJ/Bc=", + "requires": { + "to-no-case": "^1.0.0" + } + }, "toidentifier": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/toidentifier/-/toidentifier-1.0.0.tgz", "integrity": "sha512-yaOH/Pk/VEhBWWTlhI+qXxDFXlejDGcQipMlyxda9nthulaxLZUNcUqFxokp0vcYnvteJln5FNQDRrxj3YcbVw==" }, - "tough-cookie": { - "version": "2.4.3", - "resolved": "https://registry.npmjs.org/tough-cookie/-/tough-cookie-2.4.3.tgz", - "integrity": "sha512-Q5srk/4vDM54WJsJio3XNn6K2sCG+CQ8G5Wz6bZhRZoAe/+TxjWB/GlFAnYEbkYVlON9FMk/fE3h2RLpPXo4lQ==", - "requires": { - "psl": "^1.1.24", - "punycode": "^1.4.1" - }, - "dependencies": { - "punycode": { - "version": "1.4.1", - "resolved": "https://registry.npmjs.org/punycode/-/punycode-1.4.1.tgz", - "integrity": "sha512-jmYNElW7yvO7TV33CjSmvSiE2yco3bV2czu/OzDKdMNVZQWfxCblURLhf+47syQRBntjfLdd/H0egrzIG+oaFQ==" - } - } - }, "tsconfig-paths": { "version": "3.9.0", "resolved": "https://registry.npmjs.org/tsconfig-paths/-/tsconfig-paths-3.9.0.tgz", @@ -5004,9 +5719,9 @@ "integrity": "sha512-pMZTvIkT1d+TFGvDOqodOclx0QWkkgi6Tdoa8gC8ffGAAqz9pzPTZWAybbsHHoED/ztMtkv/VoYTYyShUn81hA==" }, "uuid": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/uuid/-/uuid-3.0.0.tgz", - "integrity": "sha512-rqE1LoOVLv3QrZMjb4NkF5UWlkurCfPyItVnFPNKDDGkHw4dQUdE4zMcLqx28+0Kcf3+bnUk4PisaiRJT4aiaQ==" + "version": "7.0.3", + "resolved": "https://registry.npmjs.org/uuid/-/uuid-7.0.3.tgz", + "integrity": "sha512-DPSke0pXhTZgoF/d+WSt2QaKMCFSfx7QegxEWT+JOuHF5aWrKEn0G+ztjuJg/gG8/ItK+rbPCD/yNv8yyih6Cg==" }, "v8-compile-cache": { "version": "2.1.1", @@ -5113,6 +5828,11 @@ } } }, + "walkdir": { + "version": "0.4.1", + "resolved": "https://registry.npmjs.org/walkdir/-/walkdir-0.4.1.tgz", + "integrity": "sha512-3eBwRyEln6E1MSzcxcVpQIhRG8Q1jLvEqRmCZqS3dsfXEDR/AhOF4d+jHg1qvDCpYaVRZjENPQyrVxAkQqxPgQ==" + }, "which": { "version": "1.3.1", "resolved": "https://registry.npmjs.org/which/-/which-1.3.1.tgz", diff --git a/services/real-time/package.json b/services/real-time/package.json index a1de6fad59..f928169ab6 100644 --- a/services/real-time/package.json +++ b/services/real-time/package.json @@ -27,7 +27,7 @@ "cookie-parser": "^1.4.5", "express": "^4.17.1", "express-session": "^1.17.1", - "logger-sharelatex": "^1.7.0", + "logger-sharelatex": "^2.2.0", "metrics-sharelatex": "^2.6.2", "redis-sharelatex": "^1.0.13", "request": "^2.88.2", From ee59056c6e6170e561960be8723980d7426a4d17 Mon Sep 17 00:00:00 2001 From: Jakob Ackermann Date: Thu, 13 Aug 2020 12:12:12 +0100 Subject: [PATCH 408/491] [misc] forcefully disconnect stale clients from shutdown process --- services/real-time/app.js | 15 ++++++++++++++- services/real-time/app/js/DrainManager.js | 14 ++++++++++---- services/real-time/config/settings.defaults.js | 10 ++++++++++ 3 files changed, 34 insertions(+), 5 deletions(-) diff --git a/services/real-time/app.js b/services/real-time/app.js index c41743e597..f2f756c21c 100644 --- a/services/real-time/app.js +++ b/services/real-time/app.js @@ -155,7 +155,20 @@ function drainAndShutdown(signal) { { signal }, `received interrupt, starting drain over ${shutdownDrainTimeWindow} mins` ) - DrainManager.startDrainTimeWindow(io, shutdownDrainTimeWindow) + DrainManager.startDrainTimeWindow(io, shutdownDrainTimeWindow, () => { + setTimeout(() => { + const staleClients = io.sockets.clients() + if (staleClients.length !== 0) { + logger.warn( + { staleClients: staleClients.map((client) => client.id) }, + 'forcefully disconnecting stale clients' + ) + staleClients.forEach((client) => { + client.disconnect() + }) + } + }, Settings.gracefulReconnectTimeoutMs) + }) shutdownCleanly(signal) }, statusCheckInterval) } diff --git a/services/real-time/app/js/DrainManager.js b/services/real-time/app/js/DrainManager.js index c605957160..06885f1d28 100644 --- a/services/real-time/app/js/DrainManager.js +++ b/services/real-time/app/js/DrainManager.js @@ -1,13 +1,13 @@ const logger = require('logger-sharelatex') module.exports = { - startDrainTimeWindow(io, minsToDrain) { + startDrainTimeWindow(io, minsToDrain, callback) { const drainPerMin = io.sockets.clients().length / minsToDrain // enforce minimum drain rate - this.startDrain(io, Math.max(drainPerMin / 60, 4)) + this.startDrain(io, Math.max(drainPerMin / 60, 4), callback) }, - startDrain(io, rate) { + startDrain(io, rate, callback) { // Clear out any old interval clearInterval(this.interval) logger.log({ rate }, 'starting drain') @@ -24,7 +24,11 @@ module.exports = { pollingInterval = 1000 } this.interval = setInterval(() => { - this.reconnectNClients(io, rate) + const requestedAllClientsToReconnect = this.reconnectNClients(io, rate) + if (requestedAllClientsToReconnect && callback) { + callback() + callback = undefined + } }, pollingInterval) }, @@ -48,6 +52,8 @@ module.exports = { } if (drainedCount < N) { logger.log('All clients have been told to reconnectGracefully') + return true } + return false } } diff --git a/services/real-time/config/settings.defaults.js b/services/real-time/config/settings.defaults.js index cbf6d3be82..5c0d501b50 100644 --- a/services/real-time/config/settings.defaults.js +++ b/services/real-time/config/settings.defaults.js @@ -123,6 +123,16 @@ const settings = { shutdownDrainTimeWindow: process.env.SHUTDOWN_DRAIN_TIME_WINDOW || 9, + // The shutdown procedure asks clients to reconnect gracefully. + // 3rd-party/buggy clients may not act upon receiving the message and keep + // stale connections alive. We forcefully disconnect them after X ms: + gracefulReconnectTimeoutMs: + parseInt(process.env.GRACEFUL_RECONNECT_TIMEOUT_MS, 10) || + // The frontend allows actively editing users to keep the connection open + // for up-to ConnectionManager.MAX_RECONNECT_GRACEFULLY_INTERVAL=45s + // Permit an extra delay to account for slow/flaky connections. + (45 + 30) * 1000, + continualPubsubTraffic: process.env.CONTINUAL_PUBSUB_TRAFFIC || false, checkEventOrder: process.env.CHECK_EVENT_ORDER || false, From ea75b84eefedaf03847612f3e4c02834e3b700ff Mon Sep 17 00:00:00 2001 From: Jakob Ackermann Date: Thu, 13 Aug 2020 12:39:51 +0100 Subject: [PATCH 409/491] [misc] let the orchestrator handle the process restart Note that there is also the `shutdownCleanly` interval which may notice that the shutdown has completed. There is some network IO required to signal all clients the server disconnect, so we cannot run process.exit immediately. --- services/real-time/app.js | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/services/real-time/app.js b/services/real-time/app.js index f2f756c21c..485937889d 100644 --- a/services/real-time/app.js +++ b/services/real-time/app.js @@ -98,7 +98,16 @@ function healthCheck(req, res) { } }) } -app.get('/health_check', healthCheck) +app.get( + '/health_check', + (req, res, next) => { + if (Settings.shutDownComplete) { + return res.sendStatus(503) + } + next() + }, + healthCheck +) app.get('/health_check/redis', healthCheck) @@ -167,6 +176,8 @@ function drainAndShutdown(signal) { client.disconnect() }) } + // Mark the node as unhealthy. + Settings.shutDownComplete = true }, Settings.gracefulReconnectTimeoutMs) }) shutdownCleanly(signal) From 83b8b077fab629e23fc574dc3307520315887d5c Mon Sep 17 00:00:00 2001 From: Jakob Ackermann Date: Thu, 13 Aug 2020 10:32:13 +0100 Subject: [PATCH 410/491] [misc] bump socket.io and socket.io-client --- services/real-time/package-lock.json | 10 +++++----- services/real-time/package.json | 4 ++-- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/services/real-time/package-lock.json b/services/real-time/package-lock.json index 045283d72a..c0f87e0611 100644 --- a/services/real-time/package-lock.json +++ b/services/real-time/package-lock.json @@ -4537,13 +4537,13 @@ } }, "socket.io": { - "version": "https://github.com/overleaf/socket.io/archive/0.9.19-overleaf-3.tar.gz", - "integrity": "sha512-x07f3i4f3/CXaTBpUT/mH4JtYf3xjZWO0tvIB43Nn8OnzeRhMaKxhMZWcLQxfPdcaAyvafwLeM06fiDpHCaIYw==", + "version": "https://github.com/overleaf/socket.io/archive/0.9.19-overleaf-4.tar.gz", + "integrity": "sha512-jWgdvVEbPioarWhfKvmtFf9miv/TYMePwrWO+r3WlVF+07nPkWY+OK0y0Lob5shC/dTLqwyG9ajw49+ObC8s/A==", "requires": { "base64id": "0.1.0", "policyfile": "0.0.4", "redis": "0.7.3", - "socket.io-client": "https://github.com/overleaf/socket.io-client/archive/0.9.17-overleaf-2.tar.gz" + "socket.io-client": "https://github.com/overleaf/socket.io-client/archive/0.9.17-overleaf-3.tar.gz" }, "dependencies": { "redis": { @@ -4555,8 +4555,8 @@ } }, "socket.io-client": { - "version": "https://github.com/overleaf/socket.io-client/archive/0.9.17-overleaf-2.tar.gz", - "integrity": "sha512-u8jEVctc4nFrN1JfjLeFjSSxXxGYo6hHxYW4rhv6aeoiaWkTmdaUzaG1YkDfSwBG/TXmdHx32UJB4fClmz8IEg==", + "version": "https://github.com/overleaf/socket.io-client/archive/0.9.17-overleaf-3.tar.gz", + "integrity": "sha512-EtKV6qGQjG/DwMXfLAiS559f07xjPVavYZ+amYwiEZ0FUHdLObuGH3zvoxghjbI6l8gFTflQhGt1foiHIGnDKg==", "requires": { "active-x-obfuscator": "0.0.1", "uglify-js": "1.2.5", diff --git a/services/real-time/package.json b/services/real-time/package.json index a1de6fad59..fd292b12ab 100644 --- a/services/real-time/package.json +++ b/services/real-time/package.json @@ -32,8 +32,8 @@ "redis-sharelatex": "^1.0.13", "request": "^2.88.2", "settings-sharelatex": "^1.1.0", - "socket.io": "https://github.com/overleaf/socket.io/archive/0.9.19-overleaf-3.tar.gz", - "socket.io-client": "https://github.com/overleaf/socket.io-client/archive/0.9.17-overleaf-2.tar.gz" + "socket.io": "https://github.com/overleaf/socket.io/archive/0.9.19-overleaf-4.tar.gz", + "socket.io-client": "https://github.com/overleaf/socket.io-client/archive/0.9.17-overleaf-3.tar.gz" }, "devDependencies": { "bunyan": "~0.22.3", From c7b9dadd16d78af089a031582d0e18a513789edc Mon Sep 17 00:00:00 2001 From: Jakob Ackermann Date: Fri, 21 Aug 2020 12:45:30 +0100 Subject: [PATCH 411/491] [misc] increase the timeout for unit tests -- CI boxes are too slow --- services/real-time/package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/services/real-time/package.json b/services/real-time/package.json index 144608eaae..4a7acfc1cd 100644 --- a/services/real-time/package.json +++ b/services/real-time/package.json @@ -11,7 +11,7 @@ "start": "node $NODE_APP_OPTIONS app.js", "test:acceptance:_run": "mocha --recursive --reporter spec --timeout 15000 --exit $@ test/acceptance/js", "test:acceptance": "npm run test:acceptance:_run -- --grep=$MOCHA_GREP", - "test:unit:_run": "mocha --recursive --reporter spec $@ test/unit/js", + "test:unit:_run": "mocha --recursive --reporter spec --timeout 5000 $@ test/unit/js", "test:unit": "npm run test:unit:_run -- --grep=$MOCHA_GREP", "nodemon": "nodemon --config nodemon.json", "lint": "node_modules/.bin/eslint --max-warnings 0 .", From f82177a46a3609f72029f10d38566886fcdaac70 Mon Sep 17 00:00:00 2001 From: Jakob Ackermann Date: Thu, 20 Aug 2020 10:41:16 +0100 Subject: [PATCH 412/491] [Errors] migrate to OError --- services/real-time/app/js/Errors.js | 8 ++++---- services/real-time/app/js/Router.js | 4 ++-- services/real-time/package.json | 1 + services/real-time/test/unit/js/WebApiManagerTests.js | 4 +++- 4 files changed, 10 insertions(+), 7 deletions(-) diff --git a/services/real-time/app/js/Errors.js b/services/real-time/app/js/Errors.js index bdda1b4c21..06bf33cf5d 100644 --- a/services/real-time/app/js/Errors.js +++ b/services/real-time/app/js/Errors.js @@ -1,8 +1,8 @@ -class CodedError extends Error { +const OError = require('@overleaf/o-error') + +class CodedError extends OError { constructor(message, code) { - super(message) - this.name = this.constructor.name - this.code = code + super(message, { code }) } } diff --git a/services/real-time/app/js/Router.js b/services/real-time/app/js/Router.js index ae10d12e2b..87c1991d4c 100644 --- a/services/real-time/app/js/Router.js +++ b/services/real-time/app/js/Router.js @@ -33,8 +33,8 @@ module.exports = Router = { attrs.client_id = client.id attrs.err = error if (error.name === 'CodedError') { - logger.warn(attrs, error.message, { code: error.code }) - const serializedError = { message: error.message, code: error.code } + logger.warn(attrs, error.message) + const serializedError = { message: error.message, code: error.info.code } callback(serializedError) } else if (error.message === 'unexpected arguments') { // the payload might be very large, put it on level info diff --git a/services/real-time/package.json b/services/real-time/package.json index 4a7acfc1cd..0fab05972f 100644 --- a/services/real-time/package.json +++ b/services/real-time/package.json @@ -19,6 +19,7 @@ "format:fix": "node_modules/.bin/prettier-eslint $PWD'/**/*.js' --write" }, "dependencies": { + "@overleaf/o-error": "^3.0.0", "async": "^0.9.0", "base64id": "0.1.0", "basic-auth-connect": "^1.0.0", diff --git a/services/real-time/test/unit/js/WebApiManagerTests.js b/services/real-time/test/unit/js/WebApiManagerTests.js index 2d792b563b..52c6da137a 100644 --- a/services/real-time/test/unit/js/WebApiManagerTests.js +++ b/services/real-time/test/unit/js/WebApiManagerTests.js @@ -152,7 +152,9 @@ describe('WebApiManager', function () { .calledWith( sinon.match({ message: 'rate-limit hit when joining project', - code: 'TooManyRequests' + info: { + code: 'TooManyRequests' + } }) ) .should.equal(true) From 5950b26a42bc8e977d77eaaf054f2f907810ddca Mon Sep 17 00:00:00 2001 From: Jakob Ackermann Date: Thu, 20 Aug 2020 11:03:34 +0100 Subject: [PATCH 413/491] [SafeJsonParse] migrate to OError and use a new DataTooLargeToParseError --- services/real-time/app/js/Errors.js | 11 ++++++++++- services/real-time/app/js/SafeJsonParse.js | 8 ++------ services/real-time/test/unit/js/SafeJsonParseTest.js | 2 +- 3 files changed, 13 insertions(+), 8 deletions(-) diff --git a/services/real-time/app/js/Errors.js b/services/real-time/app/js/Errors.js index 06bf33cf5d..12c644c839 100644 --- a/services/real-time/app/js/Errors.js +++ b/services/real-time/app/js/Errors.js @@ -6,4 +6,13 @@ class CodedError extends OError { } } -module.exports = { CodedError } +class DataTooLargeToParseError extends OError { + constructor(data) { + super('data too large to parse', { + head: data.slice(0, 1024), + length: data.length + }) + } +} + +module.exports = { CodedError, DataTooLargeToParseError } diff --git a/services/real-time/app/js/SafeJsonParse.js b/services/real-time/app/js/SafeJsonParse.js index 982fd9d424..a8a3afae4d 100644 --- a/services/real-time/app/js/SafeJsonParse.js +++ b/services/real-time/app/js/SafeJsonParse.js @@ -1,14 +1,10 @@ const Settings = require('settings-sharelatex') -const logger = require('logger-sharelatex') +const { DataTooLargeToParseError } = require('./Errors') module.exports = { parse(data, callback) { if (data.length > Settings.maxUpdateSize) { - logger.error( - { head: data.slice(0, 1024), length: data.length }, - 'data too large to parse' - ) - return callback(new Error('data too large to parse')) + return callback(new DataTooLargeToParseError(data)) } let parsed try { diff --git a/services/real-time/test/unit/js/SafeJsonParseTest.js b/services/real-time/test/unit/js/SafeJsonParseTest.js index 4fb558a6b0..51a60dea23 100644 --- a/services/real-time/test/unit/js/SafeJsonParseTest.js +++ b/services/real-time/test/unit/js/SafeJsonParseTest.js @@ -50,7 +50,7 @@ describe('SafeJsonParse', function () { const data = `{\"foo\": \"${big_blob}\"}` this.Settings.maxUpdateSize = 2 * 1024 return this.SafeJsonParse.parse(data, (error, parsed) => { - this.logger.error.called.should.equal(true) + this.logger.error.called.should.equal(false) expect(error).to.exist return done() }) From af50f9b02c768dbd7a766916ade7f5d1c4eaf54e Mon Sep 17 00:00:00 2001 From: Jakob Ackermann Date: Thu, 20 Aug 2020 11:16:26 +0100 Subject: [PATCH 414/491] [DocumentUpdaterManager] use a new UpdateTooLargeError --- services/real-time/app/js/DocumentUpdaterManager.js | 5 ++--- services/real-time/app/js/Errors.js | 8 +++++++- services/real-time/app/js/WebsocketController.js | 2 +- .../real-time/test/unit/js/DocumentUpdaterManagerTests.js | 2 +- .../real-time/test/unit/js/WebsocketControllerTests.js | 4 ++-- 5 files changed, 13 insertions(+), 8 deletions(-) diff --git a/services/real-time/app/js/DocumentUpdaterManager.js b/services/real-time/app/js/DocumentUpdaterManager.js index 3fd09d7b1d..8a2ebda1f9 100644 --- a/services/real-time/app/js/DocumentUpdaterManager.js +++ b/services/real-time/app/js/DocumentUpdaterManager.js @@ -6,6 +6,7 @@ const _ = require('underscore') const logger = require('logger-sharelatex') const settings = require('settings-sharelatex') const metrics = require('metrics-sharelatex') +const { UpdateTooLargeError } = require('./Errors') const rclient = require('redis-sharelatex').createClient( settings.redis.documentupdater @@ -125,9 +126,7 @@ const DocumentUpdaterManager = { const updateSize = jsonChange.length if (updateSize > settings.maxUpdateSize) { - const error = new Error('update is too large') - error.updateSize = updateSize - return callback(error) + return callback(new UpdateTooLargeError(updateSize)) } // record metric for each update added to queue diff --git a/services/real-time/app/js/Errors.js b/services/real-time/app/js/Errors.js index 12c644c839..56034023a0 100644 --- a/services/real-time/app/js/Errors.js +++ b/services/real-time/app/js/Errors.js @@ -15,4 +15,10 @@ class DataTooLargeToParseError extends OError { } } -module.exports = { CodedError, DataTooLargeToParseError } +class UpdateTooLargeError extends OError { + constructor(updateSize) { + super('update is too large', { updateSize }) + } +} + +module.exports = { CodedError, DataTooLargeToParseError, UpdateTooLargeError } diff --git a/services/real-time/app/js/WebsocketController.js b/services/real-time/app/js/WebsocketController.js index 8867320dbe..a0c33f5cbe 100644 --- a/services/real-time/app/js/WebsocketController.js +++ b/services/real-time/app/js/WebsocketController.js @@ -509,7 +509,7 @@ module.exports = WebsocketController = { function (error) { if ((error && error.message) === 'update is too large') { metrics.inc('update_too_large') - const { updateSize } = error + const { updateSize } = error.info logger.warn( { user_id, project_id, doc_id, updateSize }, 'update is too large' diff --git a/services/real-time/test/unit/js/DocumentUpdaterManagerTests.js b/services/real-time/test/unit/js/DocumentUpdaterManagerTests.js index dc42b52140..2ded504051 100644 --- a/services/real-time/test/unit/js/DocumentUpdaterManagerTests.js +++ b/services/real-time/test/unit/js/DocumentUpdaterManagerTests.js @@ -346,7 +346,7 @@ describe('DocumentUpdaterManager', function () { }) it('should add the size to the error', function () { - return this.callback.args[0][0].updateSize.should.equal(7782422) + return this.callback.args[0][0].info.updateSize.should.equal(7782422) }) return it('should not push the change onto the pending-updates-list queue', function () { diff --git a/services/real-time/test/unit/js/WebsocketControllerTests.js b/services/real-time/test/unit/js/WebsocketControllerTests.js index 515d407cc2..7c42dd6256 100644 --- a/services/real-time/test/unit/js/WebsocketControllerTests.js +++ b/services/real-time/test/unit/js/WebsocketControllerTests.js @@ -19,6 +19,7 @@ const { expect } = chai const modulePath = '../../../app/js/WebsocketController.js' const SandboxedModule = require('sandboxed-module') const tk = require('timekeeper') +const { UpdateTooLargeError } = require('../../../app/js/Errors') describe('WebsocketController', function () { beforeEach(function () { @@ -1507,8 +1508,7 @@ describe('WebsocketController', function () { this.client.emit = sinon.stub() this.client.ol_context.user_id = this.user_id this.client.ol_context.project_id = this.project_id - const error = new Error('update is too large') - error.updateSize = 7372835 + const error = new UpdateTooLargeError(7372835) this.DocumentUpdaterManager.queueChange = sinon .stub() .callsArgWith(3, error) From 6828becb46bcbdf1d9ce13a63f6a978e70b93774 Mon Sep 17 00:00:00 2001 From: Jakob Ackermann Date: Thu, 20 Aug 2020 11:30:22 +0100 Subject: [PATCH 415/491] [DocumentUpdaterManager] use a new NullBytesInOpError --- services/real-time/app/js/DocumentUpdaterManager.js | 9 ++------- services/real-time/app/js/Errors.js | 13 ++++++++++++- 2 files changed, 14 insertions(+), 8 deletions(-) diff --git a/services/real-time/app/js/DocumentUpdaterManager.js b/services/real-time/app/js/DocumentUpdaterManager.js index 8a2ebda1f9..167bb000a4 100644 --- a/services/real-time/app/js/DocumentUpdaterManager.js +++ b/services/real-time/app/js/DocumentUpdaterManager.js @@ -6,7 +6,7 @@ const _ = require('underscore') const logger = require('logger-sharelatex') const settings = require('settings-sharelatex') const metrics = require('metrics-sharelatex') -const { UpdateTooLargeError } = require('./Errors') +const { NullBytesInOpError, UpdateTooLargeError } = require('./Errors') const rclient = require('redis-sharelatex').createClient( settings.redis.documentupdater @@ -116,12 +116,7 @@ const DocumentUpdaterManager = { const jsonChange = JSON.stringify(change) if (jsonChange.indexOf('\u0000') !== -1) { // memory corruption check - const error = new Error('null bytes found in op') - logger.error( - { err: error, project_id, doc_id, jsonChange }, - error.message - ) - return callback(error) + return callback(new NullBytesInOpError(jsonChange)) } const updateSize = jsonChange.length diff --git a/services/real-time/app/js/Errors.js b/services/real-time/app/js/Errors.js index 56034023a0..dc90af856a 100644 --- a/services/real-time/app/js/Errors.js +++ b/services/real-time/app/js/Errors.js @@ -15,10 +15,21 @@ class DataTooLargeToParseError extends OError { } } +class NullBytesInOpError extends OError { + constructor(jsonChange) { + super('null bytes found in op', { jsonChange }) + } +} + class UpdateTooLargeError extends OError { constructor(updateSize) { super('update is too large', { updateSize }) } } -module.exports = { CodedError, DataTooLargeToParseError, UpdateTooLargeError } +module.exports = { + CodedError, + DataTooLargeToParseError, + NullBytesInOpError, + UpdateTooLargeError +} From de518ea4eb27a96accc12c69dc11d6cbf5449b57 Mon Sep 17 00:00:00 2001 From: Jakob Ackermann Date: Thu, 20 Aug 2020 11:38:10 +0100 Subject: [PATCH 416/491] [SessionSockets] use a new MissingSessionError --- services/real-time/app/js/Errors.js | 7 +++++++ services/real-time/app/js/SessionSockets.js | 3 ++- 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/services/real-time/app/js/Errors.js b/services/real-time/app/js/Errors.js index dc90af856a..7a039c54c1 100644 --- a/services/real-time/app/js/Errors.js +++ b/services/real-time/app/js/Errors.js @@ -15,6 +15,12 @@ class DataTooLargeToParseError extends OError { } } +class MissingSessionError extends OError { + constructor() { + super('could not look up session by key') + } +} + class NullBytesInOpError extends OError { constructor(jsonChange) { super('null bytes found in op', { jsonChange }) @@ -30,6 +36,7 @@ class UpdateTooLargeError extends OError { module.exports = { CodedError, DataTooLargeToParseError, + MissingSessionError, NullBytesInOpError, UpdateTooLargeError } diff --git a/services/real-time/app/js/SessionSockets.js b/services/real-time/app/js/SessionSockets.js index 4ade959829..84f87bf872 100644 --- a/services/real-time/app/js/SessionSockets.js +++ b/services/real-time/app/js/SessionSockets.js @@ -1,7 +1,8 @@ const { EventEmitter } = require('events') +const { MissingSessionError } = require('./Errors') module.exports = function (io, sessionStore, cookieParser, cookieName) { - const missingSessionError = new Error('could not look up session by key') + const missingSessionError = new MissingSessionError() const sessionSockets = new EventEmitter() function next(error, socket, session) { From a8c51de5105307bb19c8ffc2a8edbfbde6053ae3 Mon Sep 17 00:00:00 2001 From: Jakob Ackermann Date: Thu, 20 Aug 2020 11:50:35 +0100 Subject: [PATCH 417/491] [AuthorizationManager] use a new NotAuthorizedError --- services/real-time/app/js/AuthorizationManager.js | 6 ++++-- services/real-time/app/js/Errors.js | 7 +++++++ 2 files changed, 11 insertions(+), 2 deletions(-) diff --git a/services/real-time/app/js/AuthorizationManager.js b/services/real-time/app/js/AuthorizationManager.js index a545594479..0c37ac1423 100644 --- a/services/real-time/app/js/AuthorizationManager.js +++ b/services/real-time/app/js/AuthorizationManager.js @@ -1,6 +1,8 @@ /* eslint-disable camelcase, */ +const { NotAuthorizedError } = require('./Errors') + let AuthorizationManager module.exports = AuthorizationManager = { assertClientCanViewProject(client, callback) { @@ -23,7 +25,7 @@ module.exports = AuthorizationManager = { if (allowedLevels.includes(client.ol_context.privilege_level)) { callback(null) } else { - callback(new Error('not authorized')) + callback(new NotAuthorizedError()) } }, @@ -49,7 +51,7 @@ module.exports = AuthorizationManager = { if (client.ol_context[`doc:${doc_id}`] === 'allowed') { callback(null) } else { - callback(new Error('not authorized')) + callback(new NotAuthorizedError()) } }, diff --git a/services/real-time/app/js/Errors.js b/services/real-time/app/js/Errors.js index 7a039c54c1..57190b75e5 100644 --- a/services/real-time/app/js/Errors.js +++ b/services/real-time/app/js/Errors.js @@ -21,6 +21,12 @@ class MissingSessionError extends OError { } } +class NotAuthorizedError extends OError { + constructor() { + super('not authorized') + } +} + class NullBytesInOpError extends OError { constructor(jsonChange) { super('null bytes found in op', { jsonChange }) @@ -37,6 +43,7 @@ module.exports = { CodedError, DataTooLargeToParseError, MissingSessionError, + NotAuthorizedError, NullBytesInOpError, UpdateTooLargeError } From 59c4c884a5a84e8eca772b787e85c710876d9e78 Mon Sep 17 00:00:00 2001 From: Jakob Ackermann Date: Thu, 20 Aug 2020 11:50:43 +0100 Subject: [PATCH 418/491] [WebsocketController] use the new NotAuthorizedError --- services/real-time/app/js/WebsocketController.js | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/services/real-time/app/js/WebsocketController.js b/services/real-time/app/js/WebsocketController.js index a0c33f5cbe..a695d02b96 100644 --- a/services/real-time/app/js/WebsocketController.js +++ b/services/real-time/app/js/WebsocketController.js @@ -9,6 +9,7 @@ const DocumentUpdaterManager = require('./DocumentUpdaterManager') const ConnectedUsersManager = require('./ConnectedUsersManager') const WebsocketLoadBalancer = require('./WebsocketLoadBalancer') const RoomManager = require('./RoomManager') +const { NotAuthorizedError } = require('./Errors') let WebsocketController module.exports = WebsocketController = { @@ -48,12 +49,7 @@ module.exports = WebsocketController = { } if (!privilegeLevel) { - const err = new Error('not authorized') - logger.warn( - { err, project_id, user_id, client_id: client.id }, - 'user is not authorized to join project' - ) - return callback(err) + return callback(new NotAuthorizedError()) } client.ol_context = {} From 68bc9d0d23ccc7c2580d1ede900b1c45e0f75677 Mon Sep 17 00:00:00 2001 From: Jakob Ackermann Date: Thu, 20 Aug 2020 12:03:54 +0100 Subject: [PATCH 419/491] [WebApiManager] use a new WebApiRequestFailedError --- services/real-time/app/js/Errors.js | 9 ++++++++- services/real-time/app/js/WebApiManager.js | 8 ++------ services/real-time/test/unit/js/WebApiManagerTests.js | 5 ++++- 3 files changed, 14 insertions(+), 8 deletions(-) diff --git a/services/real-time/app/js/Errors.js b/services/real-time/app/js/Errors.js index 57190b75e5..5b41334fb3 100644 --- a/services/real-time/app/js/Errors.js +++ b/services/real-time/app/js/Errors.js @@ -39,11 +39,18 @@ class UpdateTooLargeError extends OError { } } +class WebApiRequestFailedError extends OError { + constructor(statusCode) { + super('non-success status code from web', { statusCode }) + } +} + module.exports = { CodedError, DataTooLargeToParseError, MissingSessionError, NotAuthorizedError, NullBytesInOpError, - UpdateTooLargeError + UpdateTooLargeError, + WebApiRequestFailedError } diff --git a/services/real-time/app/js/WebApiManager.js b/services/real-time/app/js/WebApiManager.js index 62020a98ab..dffcca51c8 100644 --- a/services/real-time/app/js/WebApiManager.js +++ b/services/real-time/app/js/WebApiManager.js @@ -4,7 +4,7 @@ const request = require('request') const settings = require('settings-sharelatex') const logger = require('logger-sharelatex') -const { CodedError } = require('./Errors') +const { CodedError, WebApiRequestFailedError } = require('./Errors') module.exports = { joinProject(project_id, user, callback) { @@ -57,11 +57,7 @@ module.exports = { ) ) } else { - err = new Error( - `non-success status code from web: ${response.statusCode}` - ) - logger.error({ err, project_id, user_id }, 'error accessing web api') - callback(err) + callback(new WebApiRequestFailedError(response.statusCode)) } } ) diff --git a/services/real-time/test/unit/js/WebApiManagerTests.js b/services/real-time/test/unit/js/WebApiManagerTests.js index 52c6da137a..4435bc14f9 100644 --- a/services/real-time/test/unit/js/WebApiManagerTests.js +++ b/services/real-time/test/unit/js/WebApiManagerTests.js @@ -106,7 +106,10 @@ describe('WebApiManager', function () { return it('should call the callback with an error', function () { return this.callback .calledWith( - sinon.match({ message: 'non-success status code from web: 500' }) + sinon.match({ + message: 'non-success status code from web', + info: { statusCode: 500 } + }) ) .should.equal(true) }) From 02a23822647433906731501b41bd9fa310c9adba Mon Sep 17 00:00:00 2001 From: Jakob Ackermann Date: Thu, 20 Aug 2020 12:14:02 +0100 Subject: [PATCH 420/491] [WebApiManager] use a new CorruptedJoinProjectResponseError --- services/real-time/app/js/Errors.js | 7 +++++++ services/real-time/app/js/WebApiManager.js | 14 ++++++-------- 2 files changed, 13 insertions(+), 8 deletions(-) diff --git a/services/real-time/app/js/Errors.js b/services/real-time/app/js/Errors.js index 5b41334fb3..ebcf1d1d81 100644 --- a/services/real-time/app/js/Errors.js +++ b/services/real-time/app/js/Errors.js @@ -6,6 +6,12 @@ class CodedError extends OError { } } +class CorruptedJoinProjectResponseError extends OError { + constructor() { + super('no data returned from joinProject request') + } +} + class DataTooLargeToParseError extends OError { constructor(data) { super('data too large to parse', { @@ -47,6 +53,7 @@ class WebApiRequestFailedError extends OError { module.exports = { CodedError, + CorruptedJoinProjectResponseError, DataTooLargeToParseError, MissingSessionError, NotAuthorizedError, diff --git a/services/real-time/app/js/WebApiManager.js b/services/real-time/app/js/WebApiManager.js index dffcca51c8..a39d9dd832 100644 --- a/services/real-time/app/js/WebApiManager.js +++ b/services/real-time/app/js/WebApiManager.js @@ -4,7 +4,11 @@ const request = require('request') const settings = require('settings-sharelatex') const logger = require('logger-sharelatex') -const { CodedError, WebApiRequestFailedError } = require('./Errors') +const { + CodedError, + CorruptedJoinProjectResponseError, + WebApiRequestFailedError +} = require('./Errors') module.exports = { joinProject(project_id, user, callback) { @@ -32,15 +36,9 @@ module.exports = { if (error) { return callback(error) } - let err if (response.statusCode >= 200 && response.statusCode < 300) { if (!(data && data.project)) { - err = new Error('no data returned from joinProject request') - logger.error( - { err, project_id, user_id }, - 'error accessing web api' - ) - return callback(err) + return callback(new CorruptedJoinProjectResponseError()) } callback( null, From 8abfdb87ff48d59fe3fe71d95d9fd6b1dd5ce721 Mon Sep 17 00:00:00 2001 From: Jakob Ackermann Date: Thu, 20 Aug 2020 12:52:59 +0100 Subject: [PATCH 421/491] [DocumentUpdaterManager] use a new DocumentUpdaterRequestFailedError --- .../app/js/DocumentUpdaterManager.js | 29 +++++++-------- services/real-time/app/js/Errors.js | 10 ++++++ .../unit/js/DocumentUpdaterManagerTests.js | 36 ++++++++++++------- 3 files changed, 45 insertions(+), 30 deletions(-) diff --git a/services/real-time/app/js/DocumentUpdaterManager.js b/services/real-time/app/js/DocumentUpdaterManager.js index 167bb000a4..1b99ed7ab4 100644 --- a/services/real-time/app/js/DocumentUpdaterManager.js +++ b/services/real-time/app/js/DocumentUpdaterManager.js @@ -6,7 +6,11 @@ const _ = require('underscore') const logger = require('logger-sharelatex') const settings = require('settings-sharelatex') const metrics = require('metrics-sharelatex') -const { NullBytesInOpError, UpdateTooLargeError } = require('./Errors') +const { + DocumentUpdaterRequestFailedError, + NullBytesInOpError, + UpdateTooLargeError +} = require('./Errors') const rclient = require('redis-sharelatex').createClient( settings.redis.documentupdater @@ -51,15 +55,9 @@ const DocumentUpdaterManager = { ) callback(err) } else { - err = new Error( - `doc updater returned a non-success status code: ${res.statusCode}` + callback( + new DocumentUpdaterRequestFailedError('getDocument', res.statusCode) ) - err.statusCode = res.statusCode - logger.error( - { err, project_id, doc_id, url }, - `doc updater returned a non-success status code: ${res.statusCode}` - ) - callback(err) } }) }, @@ -89,15 +87,12 @@ const DocumentUpdaterManager = { logger.log({ project_id }, 'deleted project from document updater') callback(null) } else { - err = new Error( - `document updater returned a failure status code: ${res.statusCode}` + callback( + new DocumentUpdaterRequestFailedError( + 'flushProjectToMongoAndDelete', + res.statusCode + ) ) - err.statusCode = res.statusCode - logger.error( - { err, project_id }, - `document updater returned failure status code: ${res.statusCode}` - ) - callback(err) } }) }, diff --git a/services/real-time/app/js/Errors.js b/services/real-time/app/js/Errors.js index ebcf1d1d81..1e6ffbd4ef 100644 --- a/services/real-time/app/js/Errors.js +++ b/services/real-time/app/js/Errors.js @@ -21,6 +21,15 @@ class DataTooLargeToParseError extends OError { } } +class DocumentUpdaterRequestFailedError extends OError { + constructor(action, statusCode) { + super('doc updater returned a non-success status code', { + action, + statusCode + }) + } +} + class MissingSessionError extends OError { constructor() { super('could not look up session by key') @@ -55,6 +64,7 @@ module.exports = { CodedError, CorruptedJoinProjectResponseError, DataTooLargeToParseError, + DocumentUpdaterRequestFailedError, MissingSessionError, NotAuthorizedError, NullBytesInOpError, diff --git a/services/real-time/test/unit/js/DocumentUpdaterManagerTests.js b/services/real-time/test/unit/js/DocumentUpdaterManagerTests.js index 2ded504051..f5023b3e30 100644 --- a/services/real-time/test/unit/js/DocumentUpdaterManagerTests.js +++ b/services/real-time/test/unit/js/DocumentUpdaterManagerTests.js @@ -163,13 +163,18 @@ describe('DocumentUpdaterManager', function () { return it('should return the callback with an error', function () { this.callback.called.should.equal(true) - const err = this.callback.getCall(0).args[0] - err.should.have.property('statusCode', 500) - err.should.have.property( - 'message', - 'doc updater returned a non-success status code: 500' - ) - return this.logger.error.called.should.equal(true) + this.callback + .calledWith( + sinon.match({ + message: 'doc updater returned a non-success status code', + info: { + action: 'getDocument', + statusCode: 500 + } + }) + ) + .should.equal(true) + this.logger.error.called.should.equal(false) }) }) }) @@ -234,12 +239,17 @@ describe('DocumentUpdaterManager', function () { return it('should return the callback with an error', function () { this.callback.called.should.equal(true) - const err = this.callback.getCall(0).args[0] - err.should.have.property('statusCode', 500) - return err.should.have.property( - 'message', - 'document updater returned a failure status code: 500' - ) + this.callback + .calledWith( + sinon.match({ + message: 'doc updater returned a non-success status code', + info: { + action: 'flushProjectToMongoAndDelete', + statusCode: 500 + } + }) + ) + .should.equal(true) }) }) }) From 4cb8cc4a85935031de4c3bf5922247e2e46ae3f7 Mon Sep 17 00:00:00 2001 From: Jakob Ackermann Date: Thu, 20 Aug 2020 12:59:52 +0100 Subject: [PATCH 422/491] [DocumentUpdaterManager] use a new ClientRequestedMissingOpsError --- .../real-time/app/js/DocumentUpdaterManager.js | 9 ++------- services/real-time/app/js/Errors.js | 9 +++++++++ .../test/unit/js/DocumentUpdaterManagerTests.js | 16 +++++++++------- 3 files changed, 20 insertions(+), 14 deletions(-) diff --git a/services/real-time/app/js/DocumentUpdaterManager.js b/services/real-time/app/js/DocumentUpdaterManager.js index 1b99ed7ab4..67ecd82c29 100644 --- a/services/real-time/app/js/DocumentUpdaterManager.js +++ b/services/real-time/app/js/DocumentUpdaterManager.js @@ -7,6 +7,7 @@ const logger = require('logger-sharelatex') const settings = require('settings-sharelatex') const metrics = require('metrics-sharelatex') const { + ClientRequestedMissingOpsError, DocumentUpdaterRequestFailedError, NullBytesInOpError, UpdateTooLargeError @@ -47,13 +48,7 @@ const DocumentUpdaterManager = { body = body || {} callback(null, body.lines, body.version, body.ranges, body.ops) } else if ([404, 422].includes(res.statusCode)) { - err = new Error('doc updater could not load requested ops') - err.statusCode = res.statusCode - logger.warn( - { err, project_id, doc_id, url, fromVersion }, - 'doc updater could not load requested ops' - ) - callback(err) + callback(new ClientRequestedMissingOpsError(res.statusCode)) } else { callback( new DocumentUpdaterRequestFailedError('getDocument', res.statusCode) diff --git a/services/real-time/app/js/Errors.js b/services/real-time/app/js/Errors.js index 1e6ffbd4ef..24c0b3d316 100644 --- a/services/real-time/app/js/Errors.js +++ b/services/real-time/app/js/Errors.js @@ -1,5 +1,13 @@ const OError = require('@overleaf/o-error') +class ClientRequestedMissingOpsError extends OError { + constructor(statusCode) { + super('doc updater could not load requested ops', { + statusCode + }) + } +} + class CodedError extends OError { constructor(message, code) { super(message, { code }) @@ -63,6 +71,7 @@ class WebApiRequestFailedError extends OError { module.exports = { CodedError, CorruptedJoinProjectResponseError, + ClientRequestedMissingOpsError, DataTooLargeToParseError, DocumentUpdaterRequestFailedError, MissingSessionError, diff --git a/services/real-time/test/unit/js/DocumentUpdaterManagerTests.js b/services/real-time/test/unit/js/DocumentUpdaterManagerTests.js index f5023b3e30..828f516364 100644 --- a/services/real-time/test/unit/js/DocumentUpdaterManagerTests.js +++ b/services/real-time/test/unit/js/DocumentUpdaterManagerTests.js @@ -136,14 +136,16 @@ describe('DocumentUpdaterManager', function () { return it('should return the callback with an error', function () { this.callback.called.should.equal(true) - const err = this.callback.getCall(0).args[0] - err.should.have.property('statusCode', statusCode) - err.should.have.property( - 'message', - 'doc updater could not load requested ops' - ) + this.callback + .calledWith( + sinon.match({ + message: 'doc updater could not load requested ops', + info: { statusCode } + }) + ) + .should.equal(true) this.logger.error.called.should.equal(false) - return this.logger.warn.called.should.equal(true) + this.logger.warn.called.should.equal(false) }) }) ) From 0462e3e4371156e7ebd7db940141406475764ac0 Mon Sep 17 00:00:00 2001 From: Jakob Ackermann Date: Thu, 20 Aug 2020 13:07:27 +0100 Subject: [PATCH 423/491] [WebsocketController] use a new NotJoinedError --- services/real-time/app/js/Errors.js | 7 +++++++ services/real-time/app/js/WebsocketController.js | 8 ++++---- 2 files changed, 11 insertions(+), 4 deletions(-) diff --git a/services/real-time/app/js/Errors.js b/services/real-time/app/js/Errors.js index 24c0b3d316..d684764b0c 100644 --- a/services/real-time/app/js/Errors.js +++ b/services/real-time/app/js/Errors.js @@ -50,6 +50,12 @@ class NotAuthorizedError extends OError { } } +class NotJoinedError extends OError { + constructor() { + super('no project_id found on client') + } +} + class NullBytesInOpError extends OError { constructor(jsonChange) { super('null bytes found in op', { jsonChange }) @@ -76,6 +82,7 @@ module.exports = { DocumentUpdaterRequestFailedError, MissingSessionError, NotAuthorizedError, + NotJoinedError, NullBytesInOpError, UpdateTooLargeError, WebApiRequestFailedError diff --git a/services/real-time/app/js/WebsocketController.js b/services/real-time/app/js/WebsocketController.js index a695d02b96..b099d8e4e4 100644 --- a/services/real-time/app/js/WebsocketController.js +++ b/services/real-time/app/js/WebsocketController.js @@ -9,7 +9,7 @@ const DocumentUpdaterManager = require('./DocumentUpdaterManager') const ConnectedUsersManager = require('./ConnectedUsersManager') const WebsocketLoadBalancer = require('./WebsocketLoadBalancer') const RoomManager = require('./RoomManager') -const { NotAuthorizedError } = require('./Errors') +const { NotAuthorizedError, NotJoinedError } = require('./Errors') let WebsocketController module.exports = WebsocketController = { @@ -158,7 +158,7 @@ module.exports = WebsocketController = { metrics.inc('editor.join-doc') const { project_id, user_id, is_restricted_user } = client.ol_context if (!project_id) { - return callback(new Error('no project_id found on client')) + return callback(new NotJoinedError()) } logger.log( { user_id, project_id, doc_id, fromVersion, client_id: client.id }, @@ -424,7 +424,7 @@ module.exports = WebsocketController = { return callback(null, []) } if (!project_id) { - return callback(new Error('no project_id found on client')) + return callback(new NotJoinedError()) } logger.log( { user_id, project_id, client_id: client.id }, @@ -459,7 +459,7 @@ module.exports = WebsocketController = { // client may have disconnected, but we can submit their update to doc-updater anyways. const { user_id, project_id } = client.ol_context if (!project_id) { - return callback(new Error('no project_id found on client')) + return callback(new NotJoinedError()) } WebsocketController._assertClientCanApplyUpdate( From 50140f785a2fbfb5e78b524b5261db95994e931a Mon Sep 17 00:00:00 2001 From: Jakob Ackermann Date: Thu, 20 Aug 2020 13:10:22 +0100 Subject: [PATCH 424/491] [WebsocketController] use a new JoinLeaveEpochMismatchError --- services/real-time/app/js/Errors.js | 7 +++++++ services/real-time/app/js/WebsocketController.js | 8 ++++++-- 2 files changed, 13 insertions(+), 2 deletions(-) diff --git a/services/real-time/app/js/Errors.js b/services/real-time/app/js/Errors.js index d684764b0c..c5c082bfac 100644 --- a/services/real-time/app/js/Errors.js +++ b/services/real-time/app/js/Errors.js @@ -38,6 +38,12 @@ class DocumentUpdaterRequestFailedError extends OError { } } +class JoinLeaveEpochMismatchError extends OError { + constructor() { + super('joinLeaveEpoch mismatch') + } +} + class MissingSessionError extends OError { constructor() { super('could not look up session by key') @@ -80,6 +86,7 @@ module.exports = { ClientRequestedMissingOpsError, DataTooLargeToParseError, DocumentUpdaterRequestFailedError, + JoinLeaveEpochMismatchError, MissingSessionError, NotAuthorizedError, NotJoinedError, diff --git a/services/real-time/app/js/WebsocketController.js b/services/real-time/app/js/WebsocketController.js index b099d8e4e4..a443a49f95 100644 --- a/services/real-time/app/js/WebsocketController.js +++ b/services/real-time/app/js/WebsocketController.js @@ -9,7 +9,11 @@ const DocumentUpdaterManager = require('./DocumentUpdaterManager') const ConnectedUsersManager = require('./ConnectedUsersManager') const WebsocketLoadBalancer = require('./WebsocketLoadBalancer') const RoomManager = require('./RoomManager') -const { NotAuthorizedError, NotJoinedError } = require('./Errors') +const { + JoinLeaveEpochMismatchError, + NotAuthorizedError, + NotJoinedError +} = require('./Errors') let WebsocketController module.exports = WebsocketController = { @@ -180,7 +184,7 @@ module.exports = WebsocketController = { } if (joinLeaveEpoch !== client.joinLeaveEpoch) { // another joinDoc or leaveDoc rpc overtook us - return callback(new Error('joinLeaveEpoch mismatch')) + return callback(new JoinLeaveEpochMismatchError()) } // ensure the per-doc applied-ops channel is subscribed before sending the // doc to the client, so that no events are missed. From 880056d397137c26b5769c45ecfff6a6d9ba7e59 Mon Sep 17 00:00:00 2001 From: Jakob Ackermann Date: Thu, 20 Aug 2020 13:12:56 +0100 Subject: [PATCH 425/491] [Router] use a new UnexpectedArgumentsError --- services/real-time/app/js/Errors.js | 7 +++++++ services/real-time/app/js/Router.js | 3 ++- 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/services/real-time/app/js/Errors.js b/services/real-time/app/js/Errors.js index c5c082bfac..562c9888ab 100644 --- a/services/real-time/app/js/Errors.js +++ b/services/real-time/app/js/Errors.js @@ -68,6 +68,12 @@ class NullBytesInOpError extends OError { } } +class UnexpectedArgumentsError extends OError { + constructor() { + super('unexpected arguments') + } +} + class UpdateTooLargeError extends OError { constructor(updateSize) { super('update is too large', { updateSize }) @@ -91,6 +97,7 @@ module.exports = { NotAuthorizedError, NotJoinedError, NullBytesInOpError, + UnexpectedArgumentsError, UpdateTooLargeError, WebApiRequestFailedError } diff --git a/services/real-time/app/js/Router.js b/services/real-time/app/js/Router.js index 87c1991d4c..5e0ae4bf76 100644 --- a/services/real-time/app/js/Router.js +++ b/services/real-time/app/js/Router.js @@ -9,6 +9,7 @@ const HttpController = require('./HttpController') const HttpApiController = require('./HttpApiController') const bodyParser = require('body-parser') const base64id = require('base64id') +const { UnexpectedArgumentsError } = require('./Errors') const basicAuth = require('basic-auth-connect') const httpAuth = basicAuth(function (user, pass) { @@ -64,7 +65,7 @@ module.exports = Router = { }, _handleInvalidArguments(client, method, args) { - const error = new Error('unexpected arguments') + const error = new UnexpectedArgumentsError() let callback = args[args.length - 1] if (typeof callback !== 'function') { callback = function () {} From fd88819eec3aa78096555336f26aa9edf541ebb2 Mon Sep 17 00:00:00 2001 From: Jakob Ackermann Date: Thu, 20 Aug 2020 14:03:03 +0100 Subject: [PATCH 426/491] [Router] _handleError: ol_context.doc_id does not exist, drop overwrite --- services/real-time/app/js/Router.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/services/real-time/app/js/Router.js b/services/real-time/app/js/Router.js index ae10d12e2b..a39cd39171 100644 --- a/services/real-time/app/js/Router.js +++ b/services/real-time/app/js/Router.js @@ -27,7 +27,7 @@ let Router module.exports = Router = { _handleError(callback, error, client, method, attrs) { attrs = attrs || {} - for (const key of ['project_id', 'doc_id', 'user_id']) { + for (const key of ['project_id', 'user_id']) { attrs[key] = client.ol_context[key] } attrs.client_id = client.id From f935b1881a00ce39e5c4df7e8b772858fee5053c Mon Sep 17 00:00:00 2001 From: Jakob Ackermann Date: Thu, 20 Aug 2020 14:05:12 +0100 Subject: [PATCH 427/491] [Router] leaveDoc: pass the doc_id into the error-context --- services/real-time/app/js/Router.js | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/services/real-time/app/js/Router.js b/services/real-time/app/js/Router.js index a39cd39171..dca3794368 100644 --- a/services/real-time/app/js/Router.js +++ b/services/real-time/app/js/Router.js @@ -274,7 +274,9 @@ module.exports = Router = { WebsocketController.leaveDoc(client, doc_id, function (err, ...args) { if (err) { - Router._handleError(callback, err, client, 'leaveDoc') + Router._handleError(callback, err, client, 'leaveDoc', { + doc_id + }) } else { callback(null, ...args) } From 8e31cc5c23dd139d6961ab841dde9a96da128812 Mon Sep 17 00:00:00 2001 From: Jakob Ackermann Date: Thu, 20 Aug 2020 16:23:57 +0100 Subject: [PATCH 428/491] [Router] _handleError: joinProject error-context may not have project_id The ol_context patch changed the priority of client context and rpc context. This lead to the (possibly missing) project_id of the client context overwriting the project_id of the rpc context. REF: f1d55c0a5437a518e9f4617473caed9ba928e648 --- services/real-time/app/js/Router.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/services/real-time/app/js/Router.js b/services/real-time/app/js/Router.js index dca3794368..4fe3eef5e0 100644 --- a/services/real-time/app/js/Router.js +++ b/services/real-time/app/js/Router.js @@ -28,7 +28,7 @@ module.exports = Router = { _handleError(callback, error, client, method, attrs) { attrs = attrs || {} for (const key of ['project_id', 'user_id']) { - attrs[key] = client.ol_context[key] + attrs[key] = attrs[key] || client.ol_context[key] } attrs.client_id = client.id attrs.err = error From 537e97be73bc4aec2bbdbef78726a8d5746b286c Mon Sep 17 00:00:00 2001 From: Jakob Ackermann Date: Thu, 20 Aug 2020 14:05:50 +0100 Subject: [PATCH 429/491] [misc] OError.tag all the errors in async contexts See the docs of OError.tag: https://github.com/overleaf/o-error#long-stack-traces-with-oerrortag (currently at 221dd902e7bfa0ee92de1ea5a3cbf3152c3ceeb4) I am tagging all errors at each async hop. Most of the controller code will only ever see already tagged errors -- or new errors created in our app code. They should have enough info that we do not need to tag them again. --- services/real-time/app/js/ChannelManager.js | 3 ++ .../real-time/app/js/ConnectedUsersManager.js | 34 +++++++------ .../app/js/DocumentUpdaterManager.js | 20 ++++---- services/real-time/app/js/RoomManager.js | 5 ++ services/real-time/app/js/SessionSockets.js | 4 ++ services/real-time/app/js/WebApiManager.js | 2 + .../real-time/app/js/WebsocketController.js | 49 ++++++------------- .../test/unit/js/WebsocketControllerTests.js | 4 +- 8 files changed, 63 insertions(+), 58 deletions(-) diff --git a/services/real-time/app/js/ChannelManager.js b/services/real-time/app/js/ChannelManager.js index e84cfe44a9..da0490ff9a 100644 --- a/services/real-time/app/js/ChannelManager.js +++ b/services/real-time/app/js/ChannelManager.js @@ -1,6 +1,7 @@ const logger = require('logger-sharelatex') const metrics = require('metrics-sharelatex') const settings = require('settings-sharelatex') +const OError = require('@overleaf/o-error') const ClientMap = new Map() // for each redis client, store a Map of subscribed channels (channelname -> subscribe promise) @@ -35,6 +36,8 @@ module.exports = { .catch(function (err) { logger.error({ channel, err }, 'failed to subscribe to channel') metrics.inc(`subscribe.failed.${baseChannel}`) + // add context for the stack-trace at the call-site + OError.tag(err, 'failed to subscribe to channel', { channel }) }) return p } diff --git a/services/real-time/app/js/ConnectedUsersManager.js b/services/real-time/app/js/ConnectedUsersManager.js index 06ea442d63..340d0706fd 100644 --- a/services/real-time/app/js/ConnectedUsersManager.js +++ b/services/real-time/app/js/ConnectedUsersManager.js @@ -5,6 +5,7 @@ const async = require('async') const Settings = require('settings-sharelatex') const logger = require('logger-sharelatex') const redis = require('redis-sharelatex') +const OError = require('@overleaf/o-error') const rclient = redis.createClient(Settings.redis.realtime) const Keys = Settings.redis.realtime.key_schema @@ -67,10 +68,7 @@ module.exports = { multi.exec(function (err) { if (err) { - logger.err( - { err, project_id, client_id }, - 'problem marking user as connected' - ) + OError.tag(err, 'problem marking user as connected') } callback(err) }) @@ -104,7 +102,12 @@ module.exports = { multi.srem(Keys.clientsInProject({ project_id }), client_id) multi.expire(Keys.clientsInProject({ project_id }), FOUR_DAYS_IN_S) multi.del(Keys.connectedUser({ project_id, client_id })) - multi.exec(callback) + multi.exec(function (err) { + if (err) { + OError.tag(err, 'problem marking user as disconnected') + } + callback(err) + }) }, _getConnectedUser(project_id, client_id, callback) { @@ -112,6 +115,12 @@ module.exports = { err, result ) { + if (err) { + OError.tag(err, 'problem fetching connected user details', { + other_client_id: client_id + }) + return callback(err) + } if (!(result && result.user_id)) { result = { connected: false, @@ -126,15 +135,10 @@ module.exports = { try { result.cursorData = JSON.parse(result.cursorData) } catch (e) { - logger.error( - { - err: e, - project_id, - client_id, - cursorData: result.cursorData - }, - 'error parsing cursorData JSON' - ) + OError.tag(e, 'error parsing cursorData JSON', { + other_client_id: client_id, + cursorData: result.cursorData + }) return callback(e) } } @@ -150,6 +154,7 @@ module.exports = { results ) { if (err) { + OError.tag(err, 'problem getting clients in project') return callback(err) } const jobs = results.map((client_id) => (cb) => @@ -157,6 +162,7 @@ module.exports = { ) async.series(jobs, function (err, users) { if (err) { + OError.tag(err, 'problem getting connected users') return callback(err) } users = users.filter( diff --git a/services/real-time/app/js/DocumentUpdaterManager.js b/services/real-time/app/js/DocumentUpdaterManager.js index 3fd09d7b1d..587b6f959d 100644 --- a/services/real-time/app/js/DocumentUpdaterManager.js +++ b/services/real-time/app/js/DocumentUpdaterManager.js @@ -3,6 +3,7 @@ */ const request = require('request') const _ = require('underscore') +const OError = require('@overleaf/o-error') const logger = require('logger-sharelatex') const settings = require('settings-sharelatex') const metrics = require('metrics-sharelatex') @@ -23,10 +24,7 @@ const DocumentUpdaterManager = { request.get(url, function (err, res, body) { timer.done() if (err) { - logger.error( - { err, url, project_id, doc_id }, - 'error getting doc from doc updater' - ) + OError.tag(err, 'error getting doc from doc updater') return callback(err) } if (res.statusCode >= 200 && res.statusCode < 300) { @@ -37,6 +35,7 @@ const DocumentUpdaterManager = { try { body = JSON.parse(body) } catch (error) { + OError.tag(error, 'error parsing doc updater response') return callback(error) } body = body || {} @@ -79,10 +78,7 @@ const DocumentUpdaterManager = { request.del(url, function (err, res) { timer.done() if (err) { - logger.error( - { err, project_id }, - 'error deleting project from document updater' - ) + OError.tag(err, 'error deleting project from document updater') callback(err) } else if (res.statusCode >= 200 && res.statusCode < 300) { logger.log({ project_id }, 'deleted project from document updater') @@ -140,9 +136,15 @@ const DocumentUpdaterManager = { error ) { if (error) { + OError.tag(error, 'error pushing update into redis') return callback(error) } - rclient.rpush('pending-updates-list', doc_key, callback) + rclient.rpush('pending-updates-list', doc_key, function (error) { + if (error) { + OError.tag(error, 'error pushing doc_id into redis') + } + callback(error) + }) }) } } diff --git a/services/real-time/app/js/RoomManager.js b/services/real-time/app/js/RoomManager.js index 81827e5e9a..549975843e 100644 --- a/services/real-time/app/js/RoomManager.js +++ b/services/real-time/app/js/RoomManager.js @@ -4,6 +4,7 @@ const logger = require('logger-sharelatex') const metrics = require('metrics-sharelatex') const { EventEmitter } = require('events') +const OError = require('@overleaf/o-error') const IdMap = new Map() // keep track of whether ids are from projects or docs const RoomEvents = new EventEmitter() // emits {project,doc}-active and {project,doc}-empty events @@ -65,6 +66,10 @@ module.exports = { logger.log({ entity, id }, 'room is now active') RoomEvents.once(`${entity}-subscribed-${id}`, function (err) { // only allow the client to join when all the relevant channels have subscribed + if (err) { + OError.tag(err, 'error joining', { entity, id }) + return callback(err) + } logger.log( { client: client.id, entity, id, beforeCount }, 'client joined new room and subscribed to channel' diff --git a/services/real-time/app/js/SessionSockets.js b/services/real-time/app/js/SessionSockets.js index 4ade959829..2edb48bb6e 100644 --- a/services/real-time/app/js/SessionSockets.js +++ b/services/real-time/app/js/SessionSockets.js @@ -1,3 +1,4 @@ +const OError = require('@overleaf/o-error') const { EventEmitter } = require('events') module.exports = function (io, sessionStore, cookieParser, cookieName) { @@ -17,6 +18,9 @@ module.exports = function (io, sessionStore, cookieParser, cookieName) { } sessionStore.get(sessionId, function (error, session) { if (error) { + OError.tag(error, 'error getting session from sessionStore', { + sessionId + }) return next(error, socket) } if (!session) { diff --git a/services/real-time/app/js/WebApiManager.js b/services/real-time/app/js/WebApiManager.js index 62020a98ab..79b0937ad5 100644 --- a/services/real-time/app/js/WebApiManager.js +++ b/services/real-time/app/js/WebApiManager.js @@ -2,6 +2,7 @@ camelcase, */ const request = require('request') +const OError = require('@overleaf/o-error') const settings = require('settings-sharelatex') const logger = require('logger-sharelatex') const { CodedError } = require('./Errors') @@ -30,6 +31,7 @@ module.exports = { }, function (error, response, data) { if (error) { + OError.tag(error, 'join project request failed') return callback(error) } let err diff --git a/services/real-time/app/js/WebsocketController.js b/services/real-time/app/js/WebsocketController.js index 8867320dbe..7a9233b6d2 100644 --- a/services/real-time/app/js/WebsocketController.js +++ b/services/real-time/app/js/WebsocketController.js @@ -1,6 +1,7 @@ /* eslint-disable camelcase, */ +const OError = require('@overleaf/o-error') const logger = require('logger-sharelatex') const metrics = require('metrics-sharelatex') const WebApiManager = require('./WebApiManager') @@ -91,7 +92,14 @@ module.exports = WebsocketController = { client.publicId, user, null, - function () {} + function (err) { + if (err) { + logger.warn( + { err, project_id, user_id, client_id: client.id }, + 'background cursor update failed' + ) + } + } ) }) }, @@ -229,17 +237,7 @@ module.exports = WebsocketController = { try { line = encodeForWebsockets(line) } catch (err) { - logger.err( - { - err, - project_id, - doc_id, - fromVersion, - line, - client_id: client.id - }, - 'error encoding line uri component' - ) + OError.tag(err, 'error encoding line uri component', { line }) return callback(err) } escapedLines.push(line) @@ -260,17 +258,9 @@ module.exports = WebsocketController = { } } } catch (err) { - logger.err( - { - err, - project_id, - doc_id, - fromVersion, - ranges, - client_id: client.id - }, - 'error encoding range uri component' - ) + OError.tag(err, 'error encoding range uri component', { + ranges + }) return callback(err) } } @@ -538,16 +528,9 @@ module.exports = WebsocketController = { } if (error) { - logger.error( - { - err: error, - project_id, - doc_id, - client_id: client.id, - version: update.v - }, - 'document was not available for update' - ) + OError.tag(error, 'document was not available for update', { + version: update.v + }) client.disconnect() } callback(error) diff --git a/services/real-time/test/unit/js/WebsocketControllerTests.js b/services/real-time/test/unit/js/WebsocketControllerTests.js index 515d407cc2..1d363123a7 100644 --- a/services/real-time/test/unit/js/WebsocketControllerTests.js +++ b/services/real-time/test/unit/js/WebsocketControllerTests.js @@ -1464,8 +1464,8 @@ describe('WebsocketController', function () { return this.client.disconnect.called.should.equal(true) }) - it('should log an error', function () { - return this.logger.error.called.should.equal(true) + it('should not log an error', function () { + return this.logger.error.called.should.equal(false) }) return it('should call the callback with the error', function () { From ee3d3b09ed72f1f78d4aaea29b3448d080b30a3f Mon Sep 17 00:00:00 2001 From: Jakob Ackermann Date: Fri, 21 Aug 2020 12:15:35 +0100 Subject: [PATCH 430/491] [misc] wrap redis errors as tagging does not work with them ioredis may reuse the error instance for multiple callbacks. E.g. when the connection to redis fails, the queue is flushed with the same MaxRetriesPerRequestError instance. --- services/real-time/app/js/ChannelManager.js | 18 ++++++++++-------- .../real-time/app/js/ConnectedUsersManager.js | 10 +++++----- .../real-time/app/js/DocumentUpdaterManager.js | 4 ++-- .../test/unit/js/ChannelManagerTests.js | 3 ++- 4 files changed, 19 insertions(+), 16 deletions(-) diff --git a/services/real-time/app/js/ChannelManager.js b/services/real-time/app/js/ChannelManager.js index da0490ff9a..8e2bb57863 100644 --- a/services/real-time/app/js/ChannelManager.js +++ b/services/real-time/app/js/ChannelManager.js @@ -23,12 +23,13 @@ module.exports = { const channel = `${baseChannel}:${id}` const actualSubscribe = function () { // subscribe is happening in the foreground and it should reject - const p = rclient.subscribe(channel) - p.finally(function () { - if (clientChannelMap.get(channel) === subscribePromise) { - clientChannelMap.delete(channel) - } - }) + return rclient + .subscribe(channel) + .finally(function () { + if (clientChannelMap.get(channel) === subscribePromise) { + clientChannelMap.delete(channel) + } + }) .then(function () { logger.log({ channel }, 'subscribed to channel') metrics.inc(`subscribe.${baseChannel}`) @@ -37,9 +38,10 @@ module.exports = { logger.error({ channel, err }, 'failed to subscribe to channel') metrics.inc(`subscribe.failed.${baseChannel}`) // add context for the stack-trace at the call-site - OError.tag(err, 'failed to subscribe to channel', { channel }) + throw new OError('failed to subscribe to channel', { + channel + }).withCause(err) }) - return p } const pendingActions = clientChannelMap.get(channel) || Promise.resolve() diff --git a/services/real-time/app/js/ConnectedUsersManager.js b/services/real-time/app/js/ConnectedUsersManager.js index 340d0706fd..ed37db1ef6 100644 --- a/services/real-time/app/js/ConnectedUsersManager.js +++ b/services/real-time/app/js/ConnectedUsersManager.js @@ -68,7 +68,7 @@ module.exports = { multi.exec(function (err) { if (err) { - OError.tag(err, 'problem marking user as connected') + err = new OError('problem marking user as connected').withCause(err) } callback(err) }) @@ -104,7 +104,7 @@ module.exports = { multi.del(Keys.connectedUser({ project_id, client_id })) multi.exec(function (err) { if (err) { - OError.tag(err, 'problem marking user as disconnected') + err = new OError('problem marking user as disconnected').withCause(err) } callback(err) }) @@ -116,9 +116,9 @@ module.exports = { result ) { if (err) { - OError.tag(err, 'problem fetching connected user details', { + err = new OError('problem fetching connected user details', { other_client_id: client_id - }) + }).withCause(err) return callback(err) } if (!(result && result.user_id)) { @@ -154,7 +154,7 @@ module.exports = { results ) { if (err) { - OError.tag(err, 'problem getting clients in project') + err = new OError('problem getting clients in project').withCause(err) return callback(err) } const jobs = results.map((client_id) => (cb) => diff --git a/services/real-time/app/js/DocumentUpdaterManager.js b/services/real-time/app/js/DocumentUpdaterManager.js index 587b6f959d..2d7f2890c6 100644 --- a/services/real-time/app/js/DocumentUpdaterManager.js +++ b/services/real-time/app/js/DocumentUpdaterManager.js @@ -136,12 +136,12 @@ const DocumentUpdaterManager = { error ) { if (error) { - OError.tag(error, 'error pushing update into redis') + error = new OError('error pushing update into redis').withCause(error) return callback(error) } rclient.rpush('pending-updates-list', doc_key, function (error) { if (error) { - OError.tag(error, 'error pushing doc_id into redis') + error = new OError('error pushing doc_id into redis').withCause(error) } callback(error) }) diff --git a/services/real-time/test/unit/js/ChannelManagerTests.js b/services/real-time/test/unit/js/ChannelManagerTests.js index 6026f6ab5c..95e9000a49 100644 --- a/services/real-time/test/unit/js/ChannelManagerTests.js +++ b/services/real-time/test/unit/js/ChannelManagerTests.js @@ -91,7 +91,8 @@ describe('ChannelManager', function () { ) p.then(() => done(new Error('should not subscribe but fail'))).catch( (err) => { - err.message.should.equal('some redis error') + err.message.should.equal('failed to subscribe to channel') + err.cause.message.should.equal('some redis error') this.ChannelManager.getClientMapEntry(this.rclient) .has('applied-ops:1234567890abcdef') .should.equal(false) From dee4749e6d8032c1914c94fc9b85dde41233b07e Mon Sep 17 00:00:00 2001 From: Jakob Ackermann Date: Thu, 27 Aug 2020 10:11:40 +0100 Subject: [PATCH 431/491] [misc] re-level log: properly silence unauthorized updateClientPosition --- services/real-time/app/js/WebsocketController.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/services/real-time/app/js/WebsocketController.js b/services/real-time/app/js/WebsocketController.js index 7b5f2af602..38a35db8e6 100644 --- a/services/real-time/app/js/WebsocketController.js +++ b/services/real-time/app/js/WebsocketController.js @@ -356,7 +356,7 @@ module.exports = WebsocketController = { cursorData.doc_id, function (error) { if (error) { - logger.warn( + logger.info( { err: error, client_id: client.id, project_id, user_id }, "silently ignoring unauthorized updateClientPosition. Client likely hasn't called joinProject yet." ) From 2ce7b36c958e70ca9070c583b97a3fdfd992b7fe Mon Sep 17 00:00:00 2001 From: Jakob Ackermann Date: Thu, 27 Aug 2020 10:18:43 +0100 Subject: [PATCH 432/491] [misc] drop duplicate log line for unauthorized applyOtUpdate calls The violation is logged in Router._handleError. --- services/real-time/app/js/WebsocketController.js | 4 ---- services/real-time/test/unit/js/WebsocketControllerTests.js | 4 ++-- 2 files changed, 2 insertions(+), 6 deletions(-) diff --git a/services/real-time/app/js/WebsocketController.js b/services/real-time/app/js/WebsocketController.js index 7b5f2af602..a7b2cd43fa 100644 --- a/services/real-time/app/js/WebsocketController.js +++ b/services/real-time/app/js/WebsocketController.js @@ -462,10 +462,6 @@ module.exports = WebsocketController = { update, function (error) { if (error) { - logger.warn( - { err: error, doc_id, client_id: client.id, version: update.v }, - 'client is not authorized to make update' - ) setTimeout( () => // Disconnect, but give the client the chance to receive the error diff --git a/services/real-time/test/unit/js/WebsocketControllerTests.js b/services/real-time/test/unit/js/WebsocketControllerTests.js index 6d69e70fe5..f210a16ec8 100644 --- a/services/real-time/test/unit/js/WebsocketControllerTests.js +++ b/services/real-time/test/unit/js/WebsocketControllerTests.js @@ -1493,8 +1493,8 @@ describe('WebsocketController', function () { // it "should disconnect the client", -> // @client.disconnect.called.should.equal true - it('should log a warning', function () { - return this.logger.warn.called.should.equal(true) + it('should not log a warning', function () { + return this.logger.warn.called.should.equal(false) }) return it('should call the callback with the error', function () { From 0647abf4333513f5f1301ef30a8abb262edede64 Mon Sep 17 00:00:00 2001 From: Jakob Ackermann Date: Thu, 27 Aug 2020 10:25:31 +0100 Subject: [PATCH 433/491] [misc] drop info-log in WebApiManager for joinProject being rate-limited The CodedError is logged at warn-level in Router._handleError. --- services/real-time/app/js/WebApiManager.js | 1 - 1 file changed, 1 deletion(-) diff --git a/services/real-time/app/js/WebApiManager.js b/services/real-time/app/js/WebApiManager.js index de293fe015..71cba0d526 100644 --- a/services/real-time/app/js/WebApiManager.js +++ b/services/real-time/app/js/WebApiManager.js @@ -49,7 +49,6 @@ module.exports = { data.isRestrictedUser ) } else if (response.statusCode === 429) { - logger.log(project_id, user_id, 'rate-limit hit when joining project') callback( new CodedError( 'rate-limit hit when joining project', From 1ff9c1e71b3f0704d63af8c2da4cfe6402655905 Mon Sep 17 00:00:00 2001 From: Jakob Ackermann Date: Thu, 27 Aug 2020 10:28:44 +0100 Subject: [PATCH 434/491] [misc] add the rpc-method into the log context in Router._handleError --- services/real-time/app/js/Router.js | 1 + 1 file changed, 1 insertion(+) diff --git a/services/real-time/app/js/Router.js b/services/real-time/app/js/Router.js index ec6de5136f..1aae3ea64e 100644 --- a/services/real-time/app/js/Router.js +++ b/services/real-time/app/js/Router.js @@ -33,6 +33,7 @@ module.exports = Router = { } attrs.client_id = client.id attrs.err = error + attrs.method = method if (error.name === 'CodedError') { logger.warn(attrs, error.message) const serializedError = { message: error.message, code: error.info.code } From d2a2b9d46e83505b0e27f0a09aecd0b0495b197f Mon Sep 17 00:00:00 2001 From: Jakob Ackermann Date: Thu, 27 Aug 2020 10:51:34 +0100 Subject: [PATCH 435/491] [misc] add tests for web replying with a 403 for joinProject --- .../test/acceptance/js/JoinProjectTests.js | 63 +++++++++++++++++++ .../acceptance/js/helpers/MockWebServer.js | 3 + .../test/unit/js/WebApiManagerTests.js | 24 +++++++ 3 files changed, 90 insertions(+) diff --git a/services/real-time/test/acceptance/js/JoinProjectTests.js b/services/real-time/test/acceptance/js/JoinProjectTests.js index 051c33d0c7..06ab8a2c5a 100644 --- a/services/real-time/test/acceptance/js/JoinProjectTests.js +++ b/services/real-time/test/acceptance/js/JoinProjectTests.js @@ -176,6 +176,69 @@ describe('joinProject', function () { }) }) + describe('when not authorized and web replies with a 403', function () { + before(function (done) { + return async.series( + [ + (cb) => { + return FixturesManager.setUpProject( + { + project_id: 'forbidden', + privilegeLevel: 'owner', + project: { + name: 'Test Project' + } + }, + (e, { project_id, user_id }) => { + this.project_id = project_id + this.user_id = user_id + cb(e) + } + ) + }, + + (cb) => { + this.client = RealTimeClient.connect() + this.client.on('connectionAccepted', cb) + }, + + (cb) => { + this.client.emit( + 'joinProject', + { project_id: this.project_id }, + (error, project, privilegeLevel, protocolVersion) => { + this.error = error + this.project = project + this.privilegeLevel = privilegeLevel + this.protocolVersion = protocolVersion + cb() + } + ) + } + ], + done + ) + }) + + it('should return an error', function () { + this.error.message.should.equal( + 'Something went wrong in real-time service' + ) + }) + + it('should not have joined the project room', function (done) { + RealTimeClient.getConnectedClient( + this.client.socket.sessionid, + (error, client) => { + expect(Array.from(client.rooms).includes(this.project_id)).to.equal( + false + ) + done() + } + ) + }) + }) + return describe('when over rate limit', function () { before(function (done) { return async.series( diff --git a/services/real-time/test/acceptance/js/helpers/MockWebServer.js b/services/real-time/test/acceptance/js/helpers/MockWebServer.js index a2cf5af50b..8cd9d4be58 100644 --- a/services/real-time/test/acceptance/js/helpers/MockWebServer.js +++ b/services/real-time/test/acceptance/js/helpers/MockWebServer.js @@ -38,6 +38,9 @@ module.exports = MockWebServer = { joinProjectRequest(req, res, next) { const { project_id } = req.params const { user_id } = req.query + if (project_id === 'forbidden') { + return res.status(403).send() + } if (project_id === 'rate-limited') { return res.status(429).send() } else { diff --git a/services/real-time/test/unit/js/WebApiManagerTests.js b/services/real-time/test/unit/js/WebApiManagerTests.js index 4435bc14f9..c2cbd3925f 100644 --- a/services/real-time/test/unit/js/WebApiManagerTests.js +++ b/services/real-time/test/unit/js/WebApiManagerTests.js @@ -91,6 +91,30 @@ describe('WebApiManager', function () { }) }) + describe('when web replies with a 403', function () { + beforeEach(function () { + this.request.post = sinon + .stub() + .callsArgWith(1, null, { statusCode: 403 }, null) + this.WebApiManager.joinProject( + this.project_id, + this.user_id, + this.callback + ) + }) + + it('should call the callback with an error', function () { + this.callback + .calledWith( + sinon.match({ + message: 'non-success status code from web', + info: { statusCode: 403 } + }) + ) + .should.equal(true) + }) + }) + describe('with an error from web', function () { beforeEach(function () { this.request.post = sinon From 884b340c759f83a8f412abd82e902e560c56b59c Mon Sep 17 00:00:00 2001 From: Jakob Ackermann Date: Thu, 27 Aug 2020 10:55:13 +0100 Subject: [PATCH 436/491] [misc] re-level log: 403 from web goes to WARN and emit 'not authorized' Users will get redirected to the login page and will see a 'restricted' page after logging in again. See frontend: ConnectionManager.reportConnectionError --- services/real-time/app/js/WebApiManager.js | 3 +++ services/real-time/test/acceptance/js/JoinProjectTests.js | 4 +--- services/real-time/test/unit/js/WebApiManagerTests.js | 3 +-- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/services/real-time/app/js/WebApiManager.js b/services/real-time/app/js/WebApiManager.js index de293fe015..bf399eff98 100644 --- a/services/real-time/app/js/WebApiManager.js +++ b/services/real-time/app/js/WebApiManager.js @@ -8,6 +8,7 @@ const logger = require('logger-sharelatex') const { CodedError, CorruptedJoinProjectResponseError, + NotAuthorizedError, WebApiRequestFailedError } = require('./Errors') @@ -56,6 +57,8 @@ module.exports = { 'TooManyRequests' ) ) + } else if (response.statusCode === 403) { + callback(new NotAuthorizedError()) } else { callback(new WebApiRequestFailedError(response.statusCode)) } diff --git a/services/real-time/test/acceptance/js/JoinProjectTests.js b/services/real-time/test/acceptance/js/JoinProjectTests.js index 06ab8a2c5a..4c8d698fa9 100644 --- a/services/real-time/test/acceptance/js/JoinProjectTests.js +++ b/services/real-time/test/acceptance/js/JoinProjectTests.js @@ -221,9 +221,7 @@ describe('joinProject', function () { }) it('should return an error', function () { - this.error.message.should.equal( - 'Something went wrong in real-time service' - ) + this.error.message.should.equal('not authorized') }) it('should not have joined the project room', function (done) { diff --git a/services/real-time/test/unit/js/WebApiManagerTests.js b/services/real-time/test/unit/js/WebApiManagerTests.js index c2cbd3925f..0866b2b585 100644 --- a/services/real-time/test/unit/js/WebApiManagerTests.js +++ b/services/real-time/test/unit/js/WebApiManagerTests.js @@ -107,8 +107,7 @@ describe('WebApiManager', function () { this.callback .calledWith( sinon.match({ - message: 'non-success status code from web', - info: { statusCode: 403 } + message: 'not authorized' }) ) .should.equal(true) From 55d938ba14b13f47199fba497841c871ecf78939 Mon Sep 17 00:00:00 2001 From: Jakob Ackermann Date: Thu, 27 Aug 2020 11:39:25 +0100 Subject: [PATCH 437/491] [misc] add tests for web replying with a 404 for joinProject --- .../test/acceptance/js/JoinProjectTests.js | 61 +++++++++++++++++++ .../acceptance/js/helpers/MockWebServer.js | 3 + .../test/unit/js/WebApiManagerTests.js | 24 ++++++++ 3 files changed, 88 insertions(+) diff --git a/services/real-time/test/acceptance/js/JoinProjectTests.js b/services/real-time/test/acceptance/js/JoinProjectTests.js index 4c8d698fa9..9e554c6d1c 100644 --- a/services/real-time/test/acceptance/js/JoinProjectTests.js +++ b/services/real-time/test/acceptance/js/JoinProjectTests.js @@ -237,6 +237,67 @@ describe('joinProject', function () { }) }) + describe('when deleted and web replies with a 404', function () { + before(function (done) { + return async.series( + [ + (cb) => { + return FixturesManager.setUpProject( + { + project_id: 'not-found', + privilegeLevel: 'owner', + project: { + name: 'Test Project' + } + }, + (e, { project_id, user_id }) => { + this.project_id = project_id + this.user_id = user_id + cb(e) + } + ) + }, + + (cb) => { + this.client = RealTimeClient.connect() + this.client.on('connectionAccepted', cb) + }, + + (cb) => { + this.client.emit( + 'joinProject', + { project_id: this.project_id }, + (error, project, privilegeLevel, protocolVersion) => { + this.error = error + this.project = project + this.privilegeLevel = privilegeLevel + this.protocolVersion = protocolVersion + cb() + } + ) + } + ], + done + ) + }) + + it('should return an error', function () { + this.error.message.should.equal('Something went wrong in real-time service') + }) + + it('should not have joined the project room', function (done) { + RealTimeClient.getConnectedClient( + this.client.socket.sessionid, + (error, client) => { + expect(Array.from(client.rooms).includes(this.project_id)).to.equal( + false + ) + done() + } + ) + }) + }) + return describe('when over rate limit', function () { before(function (done) { return async.series( diff --git a/services/real-time/test/acceptance/js/helpers/MockWebServer.js b/services/real-time/test/acceptance/js/helpers/MockWebServer.js index 8cd9d4be58..2de9e61275 100644 --- a/services/real-time/test/acceptance/js/helpers/MockWebServer.js +++ b/services/real-time/test/acceptance/js/helpers/MockWebServer.js @@ -38,6 +38,9 @@ module.exports = MockWebServer = { joinProjectRequest(req, res, next) { const { project_id } = req.params const { user_id } = req.query + if (project_id === 'not-found') { + return res.status(404).send() + } if (project_id === 'forbidden') { return res.status(403).send() } diff --git a/services/real-time/test/unit/js/WebApiManagerTests.js b/services/real-time/test/unit/js/WebApiManagerTests.js index 0866b2b585..4d5834495f 100644 --- a/services/real-time/test/unit/js/WebApiManagerTests.js +++ b/services/real-time/test/unit/js/WebApiManagerTests.js @@ -114,6 +114,30 @@ describe('WebApiManager', function () { }) }) + describe('when web replies with a 404', function () { + beforeEach(function () { + this.request.post = sinon + .stub() + .callsArgWith(1, null, { statusCode: 404 }, null) + this.WebApiManager.joinProject( + this.project_id, + this.user_id, + this.callback + ) + }) + + it('should call the callback with an error', function () { + this.callback + .calledWith( + sinon.match({ + message: 'non-success status code from web', + info: { statusCode: 404 } + }) + ) + .should.equal(true) + }) + }) + describe('with an error from web', function () { beforeEach(function () { this.request.post = sinon From 4960bdd6feaf542f8b69c428f610c4a2414beb97 Mon Sep 17 00:00:00 2001 From: Jakob Ackermann Date: Thu, 27 Aug 2020 11:47:50 +0100 Subject: [PATCH 438/491] [misc] re-level log: 404 from web -> WARN and emit 'project not found' A stale browser tab tried to join a deleted project. Emitting 'project not found'/'ProjectNotFound' will trigger a page reload in the frontend, upon web can render a 404. See frontend: ConnectionManager.joinProject callback --- services/real-time/app/js/WebApiManager.js | 2 ++ services/real-time/test/acceptance/js/JoinProjectTests.js | 2 +- services/real-time/test/unit/js/WebApiManagerTests.js | 4 ++-- 3 files changed, 5 insertions(+), 3 deletions(-) diff --git a/services/real-time/app/js/WebApiManager.js b/services/real-time/app/js/WebApiManager.js index bf399eff98..cdcbc0955f 100644 --- a/services/real-time/app/js/WebApiManager.js +++ b/services/real-time/app/js/WebApiManager.js @@ -59,6 +59,8 @@ module.exports = { ) } else if (response.statusCode === 403) { callback(new NotAuthorizedError()) + } else if (response.statusCode === 404) { + callback(new CodedError('project not found', 'ProjectNotFound')) } else { callback(new WebApiRequestFailedError(response.statusCode)) } diff --git a/services/real-time/test/acceptance/js/JoinProjectTests.js b/services/real-time/test/acceptance/js/JoinProjectTests.js index 9e554c6d1c..508e2b6614 100644 --- a/services/real-time/test/acceptance/js/JoinProjectTests.js +++ b/services/real-time/test/acceptance/js/JoinProjectTests.js @@ -282,7 +282,7 @@ describe('joinProject', function () { }) it('should return an error', function () { - this.error.message.should.equal('Something went wrong in real-time service') + this.error.code.should.equal('ProjectNotFound') }) it('should not have joined the project room', function (done) { diff --git a/services/real-time/test/unit/js/WebApiManagerTests.js b/services/real-time/test/unit/js/WebApiManagerTests.js index 4d5834495f..a366a2e75d 100644 --- a/services/real-time/test/unit/js/WebApiManagerTests.js +++ b/services/real-time/test/unit/js/WebApiManagerTests.js @@ -130,8 +130,8 @@ describe('WebApiManager', function () { this.callback .calledWith( sinon.match({ - message: 'non-success status code from web', - info: { statusCode: 404 } + message: 'project not found', + info: { code: 'ProjectNotFound' } }) ) .should.equal(true) From 72282a07df0b2d019795fc172aaa1070f7bfc896 Mon Sep 17 00:00:00 2001 From: Jakob Ackermann Date: Fri, 28 Aug 2020 11:47:45 +0100 Subject: [PATCH 439/491] [misc] socket.io: use a custom logger - forward the previously enabled log messages to our logger-module - stub the previously disabled logger methods - drop the log-level config for socket.io --- services/real-time/app.js | 18 ++++++++++++++++-- 1 file changed, 16 insertions(+), 2 deletions(-) diff --git a/services/real-time/app.js b/services/real-time/app.js index 993bb58ff6..7525ed4d11 100644 --- a/services/real-time/app.js +++ b/services/real-time/app.js @@ -23,11 +23,26 @@ const CookieParser = require('cookie-parser') const DrainManager = require('./app/js/DrainManager') const HealthCheckManager = require('./app/js/HealthCheckManager') +// NOTE: debug is invoked for every blob that is put on the wire +const socketIoLogger = { + error(...message) { + logger.info({ fromSocketIo: true, originalLevel: 'error' }, ...message) + }, + warn(...message) { + logger.info({ fromSocketIo: true, originalLevel: 'warn' }, ...message) + }, + info() {}, + debug() {}, + log() {} +} + // Set up socket.io server const app = express() const server = require('http').createServer(app) -const io = require('socket.io').listen(server) +const io = require('socket.io').listen(server, { + logger: socketIoLogger +}) // Bind to sessions const sessionStore = new RedisStore({ client: sessionRedisClient }) @@ -61,7 +76,6 @@ io.configure(function () { 'xhr-polling', 'jsonp-polling' ]) - io.set('log level', 1) }) app.get('/', (req, res) => res.send('real-time-sharelatex is alive')) From 10c6e01bd857ffa55ddb2e1c0ebbfcee5047f4c2 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Sat, 12 Sep 2020 15:08:05 +0000 Subject: [PATCH 440/491] Bump node-fetch from 2.6.0 to 2.6.1 Bumps [node-fetch](https://github.com/bitinn/node-fetch) from 2.6.0 to 2.6.1. - [Release notes](https://github.com/bitinn/node-fetch/releases) - [Changelog](https://github.com/node-fetch/node-fetch/blob/master/docs/CHANGELOG.md) - [Commits](https://github.com/bitinn/node-fetch/compare/v2.6.0...v2.6.1) Signed-off-by: dependabot[bot] --- services/real-time/package-lock.json | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/services/real-time/package-lock.json b/services/real-time/package-lock.json index 986ccbf6ca..4ba4b76ae5 100644 --- a/services/real-time/package-lock.json +++ b/services/real-time/package-lock.json @@ -3650,9 +3650,9 @@ "dev": true }, "node-fetch": { - "version": "2.6.0", - "resolved": "https://registry.npmjs.org/node-fetch/-/node-fetch-2.6.0.tgz", - "integrity": "sha512-8dG4H5ujfvFiqDmVu9fQ5bOHUC15JMjMY/Zumv26oOvvVJjM67KF8koCWIabKQ1GJIa9r2mMZscBq/TbdOcmNA==" + "version": "2.6.1", + "resolved": "https://registry.npmjs.org/node-fetch/-/node-fetch-2.6.1.tgz", + "integrity": "sha512-V4aYg89jEoVRxRb2fJdAg8FHvI7cEyYdVAh94HH0UIK8oJxUfkjlDQN9RbMx+bEjP7+ggMiFRprSti032Oipxw==" }, "node-forge": { "version": "0.8.5", From ee067053777538724fd25f111ecc22cbd3e542cc Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 16 Sep 2020 10:38:21 +0000 Subject: [PATCH 441/491] Bump lodash from 4.17.15 to 4.17.20 Bumps [lodash](https://github.com/lodash/lodash) from 4.17.15 to 4.17.20. - [Release notes](https://github.com/lodash/lodash/releases) - [Commits](https://github.com/lodash/lodash/compare/4.17.15...4.17.20) Signed-off-by: dependabot[bot] --- services/real-time/package-lock.json | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/services/real-time/package-lock.json b/services/real-time/package-lock.json index 986ccbf6ca..52fed7c792 100644 --- a/services/real-time/package-lock.json +++ b/services/real-time/package-lock.json @@ -3167,9 +3167,9 @@ } }, "lodash": { - "version": "4.17.15", - "resolved": "https://registry.npmjs.org/lodash/-/lodash-4.17.15.tgz", - "integrity": "sha512-8xOcRHvCjnocdS5cpwXQXVzmmh5e5+saE2QGoeQmbKmRS6J3VQppPOIt0MnmE+4xlZoumy0GPG0D0MVIQbNA1A==" + "version": "4.17.20", + "resolved": "https://registry.npmjs.org/lodash/-/lodash-4.17.20.tgz", + "integrity": "sha512-PlhdFcillOINfeV7Ni6oF1TAEayyZBoZ8bcshTHqOYJYlrqzRK5hagpagky5o4HfCzzd1TRkXPMFq6cKk9rGmA==" }, "lodash.at": { "version": "4.6.0", From ba3b4f7dcdf00b71872bc319917f9f76d6f9fc7b Mon Sep 17 00:00:00 2001 From: Jakob Ackermann Date: Thu, 17 Sep 2020 15:26:45 +0100 Subject: [PATCH 442/491] [misc] bump the dev-env to 3.3.4 and bump the node version to 10.22.1 --- services/real-time/.github/dependabot.yml | 6 ++++++ services/real-time/.nvmrc | 2 +- services/real-time/Dockerfile | 2 +- services/real-time/buildscript.txt | 4 ++-- services/real-time/docker-compose.yml | 4 ++-- services/real-time/package.json | 2 +- 6 files changed, 13 insertions(+), 7 deletions(-) diff --git a/services/real-time/.github/dependabot.yml b/services/real-time/.github/dependabot.yml index c6f98d843d..e2c64a3351 100644 --- a/services/real-time/.github/dependabot.yml +++ b/services/real-time/.github/dependabot.yml @@ -15,3 +15,9 @@ updates: # Block informal upgrades -- security upgrades use a separate queue. # https://docs.github.com/en/github/administering-a-repository/configuration-options-for-dependency-updates#open-pull-requests-limit open-pull-requests-limit: 0 + + # currently assign team-magma to all dependabot PRs - this may change in + # future if we reorganise teams + labels: + - "dependencies" + - "Team-Magma" diff --git a/services/real-time/.nvmrc b/services/real-time/.nvmrc index b61c07ffdd..c2f6421352 100644 --- a/services/real-time/.nvmrc +++ b/services/real-time/.nvmrc @@ -1 +1 @@ -10.21.0 +10.22.1 diff --git a/services/real-time/Dockerfile b/services/real-time/Dockerfile index 78a715757d..f0e362fca0 100644 --- a/services/real-time/Dockerfile +++ b/services/real-time/Dockerfile @@ -2,7 +2,7 @@ # Instead run bin/update_build_scripts from # https://github.com/sharelatex/sharelatex-dev-environment -FROM node:10.21.0 as base +FROM node:10.22.1 as base WORKDIR /app diff --git a/services/real-time/buildscript.txt b/services/real-time/buildscript.txt index 96813e9029..87613f469c 100644 --- a/services/real-time/buildscript.txt +++ b/services/real-time/buildscript.txt @@ -3,6 +3,6 @@ real-time --docker-repos=gcr.io/overleaf-ops --env-add= --env-pass-through= ---node-version=10.21.0 +--node-version=10.22.1 --public-repo=True ---script-version=3.3.2 +--script-version=3.3.4 diff --git a/services/real-time/docker-compose.yml b/services/real-time/docker-compose.yml index 4525e7d83d..1347882d70 100644 --- a/services/real-time/docker-compose.yml +++ b/services/real-time/docker-compose.yml @@ -6,7 +6,7 @@ version: "2.3" services: test_unit: - image: node:10.21.0 + image: node:10.22.1 volumes: - .:/app working_dir: /app @@ -18,7 +18,7 @@ services: user: node test_acceptance: - image: node:10.21.0 + image: node:10.22.1 volumes: - .:/app working_dir: /app diff --git a/services/real-time/package.json b/services/real-time/package.json index 0fab05972f..7ba01d6bee 100644 --- a/services/real-time/package.json +++ b/services/real-time/package.json @@ -11,7 +11,7 @@ "start": "node $NODE_APP_OPTIONS app.js", "test:acceptance:_run": "mocha --recursive --reporter spec --timeout 15000 --exit $@ test/acceptance/js", "test:acceptance": "npm run test:acceptance:_run -- --grep=$MOCHA_GREP", - "test:unit:_run": "mocha --recursive --reporter spec --timeout 5000 $@ test/unit/js", + "test:unit:_run": "mocha --recursive --reporter spec $@ test/unit/js", "test:unit": "npm run test:unit:_run -- --grep=$MOCHA_GREP", "nodemon": "nodemon --config nodemon.json", "lint": "node_modules/.bin/eslint --max-warnings 0 .", From f3c7619fb56ffa985fe52cb22bca2403c99a1af7 Mon Sep 17 00:00:00 2001 From: Jakob Ackermann Date: Thu, 17 Sep 2020 15:34:03 +0100 Subject: [PATCH 443/491] [misc] revert back to high timeout for unit tests --- services/real-time/package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/services/real-time/package.json b/services/real-time/package.json index 7ba01d6bee..0fab05972f 100644 --- a/services/real-time/package.json +++ b/services/real-time/package.json @@ -11,7 +11,7 @@ "start": "node $NODE_APP_OPTIONS app.js", "test:acceptance:_run": "mocha --recursive --reporter spec --timeout 15000 --exit $@ test/acceptance/js", "test:acceptance": "npm run test:acceptance:_run -- --grep=$MOCHA_GREP", - "test:unit:_run": "mocha --recursive --reporter spec $@ test/unit/js", + "test:unit:_run": "mocha --recursive --reporter spec --timeout 5000 $@ test/unit/js", "test:unit": "npm run test:unit:_run -- --grep=$MOCHA_GREP", "nodemon": "nodemon --config nodemon.json", "lint": "node_modules/.bin/eslint --max-warnings 0 .", From cc681a94f44618b2336b8f175e783f5f7e330ca2 Mon Sep 17 00:00:00 2001 From: Jakob Ackermann Date: Tue, 22 Sep 2020 14:10:23 +0100 Subject: [PATCH 444/491] [app] ignore error from writing to disconnected long-polling client --- services/real-time/app.js | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/services/real-time/app.js b/services/real-time/app.js index 7525ed4d11..9790be8f1d 100644 --- a/services/real-time/app.js +++ b/services/real-time/app.js @@ -220,9 +220,13 @@ if (Settings.shutdownDrainTimeWindow) { process.removeAllListeners('uncaughtException') process.on('uncaughtException', function (error) { if ( - ['ETIMEDOUT', 'EHOSTUNREACH', 'EPIPE', 'ECONNRESET'].includes( - error.code - ) + [ + 'ETIMEDOUT', + 'EHOSTUNREACH', + 'EPIPE', + 'ECONNRESET', + 'ERR_STREAM_WRITE_AFTER_END' + ].includes(error.code) ) { Metrics.inc('disconnected_write', 1, { status: error.code }) return logger.warn( From 4f860995d87895dc336dff3ba3ba0bff849d6aa5 Mon Sep 17 00:00:00 2001 From: Jakob Ackermann Date: Thu, 1 Oct 2020 12:25:41 +0100 Subject: [PATCH 445/491] [MockDocUpdaterServer] return a 404 when a requested doc does not exist --- .../test/acceptance/js/helpers/MockDocUpdaterServer.js | 3 +++ 1 file changed, 3 insertions(+) diff --git a/services/real-time/test/acceptance/js/helpers/MockDocUpdaterServer.js b/services/real-time/test/acceptance/js/helpers/MockDocUpdaterServer.js index f9dcc57bf7..26cc4722a0 100644 --- a/services/real-time/test/acceptance/js/helpers/MockDocUpdaterServer.js +++ b/services/real-time/test/acceptance/js/helpers/MockDocUpdaterServer.js @@ -43,6 +43,9 @@ module.exports = MockDocUpdaterServer = { if (error != null) { return next(error) } + if (!data) { + return res.sendStatus(404) + } return res.json(data) } ) From 78fbd04ef805564d76434a9ac7d7ddd384d2764e Mon Sep 17 00:00:00 2001 From: Jakob Ackermann Date: Thu, 1 Oct 2020 12:28:39 +0100 Subject: [PATCH 446/491] [MockWebServer] grant users at least the privileges of anonymous users --- services/real-time/test/acceptance/js/helpers/MockWebServer.js | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/services/real-time/test/acceptance/js/helpers/MockWebServer.js b/services/real-time/test/acceptance/js/helpers/MockWebServer.js index 2de9e61275..160996c6b9 100644 --- a/services/real-time/test/acceptance/js/helpers/MockWebServer.js +++ b/services/real-time/test/acceptance/js/helpers/MockWebServer.js @@ -31,7 +31,8 @@ module.exports = MockWebServer = { return callback( null, MockWebServer.projects[project_id], - MockWebServer.privileges[project_id][user_id] + MockWebServer.privileges[project_id][user_id] || + MockWebServer.privileges[project_id]['anonymous-user'] ) }, From 2900c60c4a04c40e42af3e14f20de156087742ef Mon Sep 17 00:00:00 2001 From: Jakob Ackermann Date: Thu, 1 Oct 2020 12:33:15 +0100 Subject: [PATCH 447/491] [FixturesManager] add a helper for setting up project and doc together --- .../acceptance/js/helpers/FixturesManager.js | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/services/real-time/test/acceptance/js/helpers/FixturesManager.js b/services/real-time/test/acceptance/js/helpers/FixturesManager.js index 3e72961cbf..b3b7aaa78d 100644 --- a/services/real-time/test/acceptance/js/helpers/FixturesManager.js +++ b/services/real-time/test/acceptance/js/helpers/FixturesManager.js @@ -109,6 +109,22 @@ module.exports = FixturesManager = { }) }, + setUpEditorSession(options, callback) { + FixturesManager.setUpProject(options, (err, detailsProject) => { + if (err) return callback(err) + + FixturesManager.setUpDoc( + detailsProject.project_id, + options, + (err, detailsDoc) => { + if (err) return callback(err) + + callback(null, Object.assign({}, detailsProject, detailsDoc)) + } + ) + }) + }, + getRandomId() { return require('crypto') .createHash('sha1') From e846192db0eb0524033b00a661a8cc24a2f13396 Mon Sep 17 00:00:00 2001 From: Jakob Ackermann Date: Thu, 1 Oct 2020 13:59:55 +0100 Subject: [PATCH 448/491] [MatrixTests] add a large testing matrix Layers/Dimensions: - users: anonymous, registered, registeredWithOwnedProject - session setup: noop, joinReadWriteProject, joinReadWriteProjectAndDoc, joinOwnProject, joinOwnProjectAndDoc - invalid requests: noop, joinProjectWithDocId, joinDocWithDocId, joinProjectWithProjectId, joinDocWithProjectId, joinProjectWithProjectIdThenJoinDocWithDocId --- .../test/acceptance/js/MatrixTests.js | 478 ++++++++++++++++++ 1 file changed, 478 insertions(+) create mode 100644 services/real-time/test/acceptance/js/MatrixTests.js diff --git a/services/real-time/test/acceptance/js/MatrixTests.js b/services/real-time/test/acceptance/js/MatrixTests.js new file mode 100644 index 0000000000..ff59c4a7ab --- /dev/null +++ b/services/real-time/test/acceptance/js/MatrixTests.js @@ -0,0 +1,478 @@ +/* +This test suite is a multi level matrix which allows us to test many cases + with all kinds of setups. + +Users/Actors are defined in USERS and are a low level entity that does connect + to a real-time pod. A typical UserItem is: + + someDescriptiveNameForTheTestSuite: { + setup(cb) { + // + const options = { client: RealTimeClient.connect(), foo: 'bar' } + cb(null, options) + } + } + +Sessions are a set of actions that a User performs in the life-cycle of a + real-time session, before they try something weird. A typical SessionItem is: + + someOtherDescriptiveNameForTheTestSuite: { + getActions(cb) { + cb(null, [ + { rpc: 'RPC_ENDPOINT', args: [...] } + ]) + } + } + +Finally there are InvalidRequests which are the weird actions I hinted on in + the Sessions section. The defined actions may be marked as 'failed' to denote + that real-time rejects them with an (for this test) expected error. + A typical InvalidRequestItem is: + + joinOwnProject: { + getActions(cb) { + cb(null, [ + { rpc: 'RPC_ENDPOINT', args: [...], failed: true } + ]) + } + } + +There is additional meta-data that UserItems and SessionItems may use to skip + certain areas of the matrix. Theses are: + +- Has the User an own project that they join as part of the Session? + UserItem: { hasOwnProject: true, setup(cb) { cb(null, { project_id, ... }) }} + SessionItem: { needsOwnProject: true } + */ +/* eslint-disable + camelcase, +*/ +const chai = require('chai') +const { expect } = chai +const async = require('async') + +const RealTimeClient = require('./helpers/RealTimeClient') +const FixturesManager = require('./helpers/FixturesManager') + +const settings = require('settings-sharelatex') +const Keys = settings.redis.documentupdater.key_schema +const redis = require('redis-sharelatex') +const rclient = redis.createClient(settings.redis.pubsub) + +function getPendingUpdates(doc_id, cb) { + rclient.lrange(Keys.pendingUpdates({ doc_id }), 0, 10, cb) +} +function cleanupPreviousUpdates(doc_id, cb) { + rclient.del(Keys.pendingUpdates({ doc_id }), cb) +} + +describe('MatrixTests', function () { + let privateProjectId, privateDocId, readWriteProjectId, readWriteDocId + + let privateClient + before(function setupPrivateProject(done) { + FixturesManager.setUpEditorSession( + { privilegeLevel: 'owner' }, + (err, { project_id, doc_id }) => { + if (err) return done(err) + privateProjectId = project_id + privateDocId = doc_id + privateClient = RealTimeClient.connect() + privateClient.on('connectionAccepted', () => { + privateClient.emit( + 'joinProject', + { project_id: privateProjectId }, + (err) => { + if (err) return done(err) + privateClient.emit('joinDoc', privateDocId, done) + } + ) + }) + } + ) + }) + + before(function setupReadWriteProject(done) { + FixturesManager.setUpEditorSession( + { + publicAccess: 'readAndWrite' + }, + (err, { project_id, doc_id }) => { + readWriteProjectId = project_id + readWriteDocId = doc_id + done(err) + } + ) + }) + + const USER_SETUP = { + anonymous: { + setup(cb) { + RealTimeClient.setSession({}, (err) => { + if (err) return cb(err) + cb(null, { + client: RealTimeClient.connect() + }) + }) + } + }, + + registered: { + setup(cb) { + const user_id = FixturesManager.getRandomId() + RealTimeClient.setSession( + { + user: { + _id: user_id, + first_name: 'Joe', + last_name: 'Bloggs' + } + }, + (err) => { + if (err) return cb(err) + cb(null, { + user_id, + client: RealTimeClient.connect() + }) + } + ) + } + }, + + registeredWithOwnedProject: { + setup(cb) { + FixturesManager.setUpEditorSession( + { privilegeLevel: 'owner' }, + (err, { project_id, user_id, doc_id }) => { + if (err) return cb(err) + cb(null, { + user_id, + project_id, + doc_id, + client: RealTimeClient.connect() + }) + } + ) + }, + hasOwnProject: true + } + } + + Object.entries(USER_SETUP).forEach((level0) => { + const [userDescription, userItem] = level0 + let options, client + + const SESSION_SETUP = { + noop: { + getActions(cb) { + cb(null, []) + }, + needsOwnProject: false + }, + + joinReadWriteProject: { + getActions(cb) { + cb(null, [ + { rpc: 'joinProject', args: [{ project_id: readWriteProjectId }] } + ]) + }, + needsOwnProject: false + }, + + joinReadWriteProjectAndDoc: { + getActions(cb) { + cb(null, [ + { rpc: 'joinProject', args: [{ project_id: readWriteProjectId }] }, + { rpc: 'joinDoc', args: [readWriteDocId] } + ]) + }, + needsOwnProject: false + }, + + joinOwnProject: { + getActions(cb) { + cb(null, [ + { rpc: 'joinProject', args: [{ project_id: options.project_id }] } + ]) + }, + needsOwnProject: true + }, + + joinOwnProjectAndDoc: { + getActions(cb) { + cb(null, [ + { rpc: 'joinProject', args: [{ project_id: options.project_id }] }, + { rpc: 'joinDoc', args: [options.doc_id] } + ]) + }, + needsOwnProject: true + } + } + + function performActions(getActions, done) { + getActions((err, actions) => { + if (err) return done(err) + + async.eachSeries( + actions, + (action, cb) => { + if (action.rpc) { + client.emit(action.rpc, ...action.args, (...returnedArgs) => { + const error = returnedArgs.shift() + if (action.fails) { + expect(error).to.exist + expect(returnedArgs).to.have.length(0) + return cb() + } + cb(error) + }) + } else { + cb(new Error('unexpected action')) + } + }, + done + ) + }) + } + + describe(userDescription, function () { + beforeEach(function userSetup(done) { + userItem.setup((err, _options) => { + if (err) return done(err) + + options = _options + client = options.client + client.on('connectionAccepted', done) + }) + }) + + Object.entries(SESSION_SETUP).forEach((level1) => { + const [sessionSetupDescription, sessionSetupItem] = level1 + const INVALID_REQUESTS = { + noop: { + getActions(cb) { + cb(null, []) + } + }, + + joinProjectWithDocId: { + getActions(cb) { + cb(null, [ + { + rpc: 'joinProject', + args: [{ project_id: privateDocId }], + fails: 1 + } + ]) + } + }, + + joinDocWithDocId: { + getActions(cb) { + cb(null, [{ rpc: 'joinDoc', args: [privateDocId], fails: 1 }]) + } + }, + + joinProjectWithProjectId: { + getActions(cb) { + cb(null, [ + { + rpc: 'joinProject', + args: [{ project_id: privateProjectId }], + fails: 1 + } + ]) + } + }, + + joinDocWithProjectId: { + getActions(cb) { + cb(null, [{ rpc: 'joinDoc', args: [privateProjectId], fails: 1 }]) + } + }, + + joinProjectWithProjectIdThenJoinDocWithDocId: { + getActions(cb) { + cb(null, [ + { + rpc: 'joinProject', + args: [{ project_id: privateProjectId }], + fails: 1 + }, + { rpc: 'joinDoc', args: [privateDocId], fails: 1 } + ]) + } + } + } + + // skip some areas of the matrix + // - some Users do not have an own project + const skip = sessionSetupItem.needsOwnProject && !userItem.hasOwnProject + + describe(sessionSetupDescription, function () { + beforeEach(function performSessionActions(done) { + if (skip) return this.skip() + performActions(sessionSetupItem.getActions, done) + }) + + Object.entries(INVALID_REQUESTS).forEach((level2) => { + const [InvalidRequestDescription, InvalidRequestItem] = level2 + describe(InvalidRequestDescription, function () { + beforeEach(function performInvalidRequests(done) { + performActions(InvalidRequestItem.getActions, done) + }) + + describe('rooms', function () { + it('should not add the user into the privateProject room', function (done) { + RealTimeClient.getConnectedClient( + client.socket.sessionid, + (error, client) => { + if (error) return done(error) + expect(client.rooms).to.not.include(privateProjectId) + done() + } + ) + }) + + it('should not add the user into the privateDoc room', function (done) { + RealTimeClient.getConnectedClient( + client.socket.sessionid, + (error, client) => { + if (error) return done(error) + expect(client.rooms).to.not.include(privateDocId) + done() + } + ) + }) + }) + + describe('receive updates', function () { + const receivedMessages = [] + beforeEach(function publishAnUpdateInRedis(done) { + const update = { + doc_id: privateDocId, + op: { + meta: { source: privateClient.publicId }, + v: 42, + doc: privateDocId, + op: [{ i: 'foo', p: 50 }] + } + } + client.on('otUpdateApplied', (update) => { + receivedMessages.push(update) + }) + privateClient.once('otUpdateApplied', () => { + setTimeout(done, 10) + }) + rclient.publish('applied-ops', JSON.stringify(update)) + }) + + it('should send nothing to client', function () { + expect(receivedMessages).to.have.length(0) + }) + }) + + describe('receive messages from web', function () { + const receivedMessages = [] + beforeEach(function publishAMessageInRedis(done) { + const event = { + room_id: privateProjectId, + message: 'removeEntity', + payload: ['foo', 'convertDocToFile'], + _id: 'web:123' + } + client.on('removeEntity', (...args) => { + receivedMessages.push(args) + }) + privateClient.once('removeEntity', () => { + setTimeout(done, 10) + }) + rclient.publish('editor-events', JSON.stringify(event)) + }) + + it('should send nothing to client', function () { + expect(receivedMessages).to.have.length(0) + }) + }) + + describe('send updates', function () { + let receivedArgs, submittedUpdates, update + + beforeEach(function cleanup(done) { + cleanupPreviousUpdates(privateDocId, done) + }) + + beforeEach(function setupUpdateFields() { + update = { + doc_id: privateDocId, + op: { + v: 43, + lastV: 42, + doc: privateDocId, + op: [{ i: 'foo', p: 50 }] + } + } + }) + + beforeEach(function sendAsUser(done) { + const userUpdate = Object.assign({}, update, { + hash: 'user' + }) + + client.emit( + 'applyOtUpdate', + privateDocId, + userUpdate, + (...args) => { + receivedArgs = args + done() + } + ) + }) + + beforeEach(function sendAsPrivateUserForReferenceOp(done) { + const privateUpdate = Object.assign({}, update, { + hash: 'private' + }) + + privateClient.emit( + 'applyOtUpdate', + privateDocId, + privateUpdate, + done + ) + }) + + beforeEach(function fetchPendingOps(done) { + getPendingUpdates(privateDocId, (err, updates) => { + submittedUpdates = updates + done(err) + }) + }) + + it('should error out trying to send', function () { + expect(receivedArgs).to.have.length(1) + expect(receivedArgs[0]).to.have.property('message') + // we are using an old version of chai: 1.9.2 + // TypeError: expect(...).to.be.oneOf is not a function + expect( + [ + 'no project_id found on client', + 'not authorized' + ].includes(receivedArgs[0].message) + ).to.equal(true) + }) + + it('should submit the private users message only', function () { + expect(submittedUpdates).to.have.length(1) + const update = JSON.parse(submittedUpdates[0]) + expect(update.hash).to.equal('private') + }) + }) + }) + }) + }) + }) + }) + }) +}) From b20cdc01228c70cffff539cedad74301f35e7cda Mon Sep 17 00:00:00 2001 From: Jakob Ackermann Date: Mon, 19 Oct 2020 15:20:57 +0100 Subject: [PATCH 449/491] [misc] bump the dev-env to 3.3.5 -- drop custom unit test timeout --- services/real-time/buildscript.txt | 2 +- services/real-time/package.json | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/services/real-time/buildscript.txt b/services/real-time/buildscript.txt index 87613f469c..35e4f6c9aa 100644 --- a/services/real-time/buildscript.txt +++ b/services/real-time/buildscript.txt @@ -5,4 +5,4 @@ real-time --env-pass-through= --node-version=10.22.1 --public-repo=True ---script-version=3.3.4 +--script-version=3.3.5 diff --git a/services/real-time/package.json b/services/real-time/package.json index 0fab05972f..7ba01d6bee 100644 --- a/services/real-time/package.json +++ b/services/real-time/package.json @@ -11,7 +11,7 @@ "start": "node $NODE_APP_OPTIONS app.js", "test:acceptance:_run": "mocha --recursive --reporter spec --timeout 15000 --exit $@ test/acceptance/js", "test:acceptance": "npm run test:acceptance:_run -- --grep=$MOCHA_GREP", - "test:unit:_run": "mocha --recursive --reporter spec --timeout 5000 $@ test/unit/js", + "test:unit:_run": "mocha --recursive --reporter spec $@ test/unit/js", "test:unit": "npm run test:unit:_run -- --grep=$MOCHA_GREP", "nodemon": "nodemon --config nodemon.json", "lint": "node_modules/.bin/eslint --max-warnings 0 .", From d6ac8c14e7a89379276f0bdca0741c76040203c8 Mon Sep 17 00:00:00 2001 From: Jakob Ackermann Date: Sun, 15 Sep 2019 14:45:20 +0100 Subject: [PATCH 450/491] [RoomManager] drop duplicate joining of entities REF: 0437e1d03f89a058f97a8884e3532a9a58b68b9d REF: 62be5e29e5232150e7063bc189c5ad8a1189f972 Signed-off-by: Jakob Ackermann --- services/real-time/app/js/RoomManager.js | 1 - 1 file changed, 1 deletion(-) diff --git a/services/real-time/app/js/RoomManager.js b/services/real-time/app/js/RoomManager.js index 549975843e..d5c0ac7e80 100644 --- a/services/real-time/app/js/RoomManager.js +++ b/services/real-time/app/js/RoomManager.js @@ -85,7 +85,6 @@ module.exports = { { client: client.id, entity, id, beforeCount }, 'client joined existing room' ) - client.join(id) callback() } }, From dea6968fa911396d308e646722c97d34de733e61 Mon Sep 17 00:00:00 2001 From: Jakob Ackermann Date: Wed, 11 Nov 2020 16:23:12 +0000 Subject: [PATCH 451/491] [misc] explicitly install underscore Use the same (outdated) version that is already in place. --- services/real-time/package.json | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/services/real-time/package.json b/services/real-time/package.json index 7ba01d6bee..2eee5a1da1 100644 --- a/services/real-time/package.json +++ b/services/real-time/package.json @@ -34,7 +34,8 @@ "request": "^2.88.2", "settings-sharelatex": "^1.1.0", "socket.io": "https://github.com/overleaf/socket.io/archive/0.9.19-overleaf-4.tar.gz", - "socket.io-client": "https://github.com/overleaf/socket.io-client/archive/0.9.17-overleaf-3.tar.gz" + "socket.io-client": "https://github.com/overleaf/socket.io-client/archive/0.9.17-overleaf-3.tar.gz", + "underscore": "1.7.0" }, "devDependencies": { "bunyan": "~0.22.3", From 15af5c79772e6580c584e684d49008b072a6025e Mon Sep 17 00:00:00 2001 From: Jakob Ackermann Date: Tue, 10 Nov 2020 11:32:06 +0000 Subject: [PATCH 452/491] [misc] bump @overleaf/redis-wrapper to version 2.0.0 --- services/real-time/app.js | 4 +- .../real-time/app/js/ConnectedUsersManager.js | 2 +- .../app/js/DocumentUpdaterManager.js | 2 +- .../real-time/app/js/RedisClientManager.js | 2 +- services/real-time/package-lock.json | 89 +++++-------------- services/real-time/package.json | 4 +- .../test/acceptance/js/ApplyUpdateTests.js | 2 +- .../test/acceptance/js/EarlyDisconnect.js | 2 +- .../test/acceptance/js/LeaveProjectTests.js | 2 +- .../test/acceptance/js/MatrixTests.js | 2 +- .../test/acceptance/js/PubSubRace.js | 2 +- .../test/acceptance/js/ReceiveUpdateTests.js | 2 +- .../acceptance/js/helpers/RealTimeClient.js | 2 +- .../unit/js/ConnectedUsersManagerTests.js | 2 +- .../unit/js/DocumentUpdaterControllerTests.js | 2 +- .../unit/js/DocumentUpdaterManagerTests.js | 2 +- 16 files changed, 38 insertions(+), 85 deletions(-) diff --git a/services/real-time/app.js b/services/real-time/app.js index 9790be8f1d..81c08a722f 100644 --- a/services/real-time/app.js +++ b/services/real-time/app.js @@ -9,7 +9,7 @@ Metrics.event_loop.monitor(logger) const express = require('express') const session = require('express-session') -const redis = require('redis-sharelatex') +const redis = require('@overleaf/redis-wrapper') if (Settings.sentry && Settings.sentry.dsn) { logger.initializeErrorReporting(Settings.sentry.dsn) } @@ -94,7 +94,7 @@ app.get('/debug/events', function (req, res) { res.send(`debug mode will log next ${Settings.debugEvents} events`) }) -const rclient = require('redis-sharelatex').createClient( +const rclient = require('@overleaf/redis-wrapper').createClient( Settings.redis.realtime ) diff --git a/services/real-time/app/js/ConnectedUsersManager.js b/services/real-time/app/js/ConnectedUsersManager.js index ed37db1ef6..6b98043ff7 100644 --- a/services/real-time/app/js/ConnectedUsersManager.js +++ b/services/real-time/app/js/ConnectedUsersManager.js @@ -4,7 +4,7 @@ const async = require('async') const Settings = require('settings-sharelatex') const logger = require('logger-sharelatex') -const redis = require('redis-sharelatex') +const redis = require('@overleaf/redis-wrapper') const OError = require('@overleaf/o-error') const rclient = redis.createClient(Settings.redis.realtime) const Keys = Settings.redis.realtime.key_schema diff --git a/services/real-time/app/js/DocumentUpdaterManager.js b/services/real-time/app/js/DocumentUpdaterManager.js index fc614b7fb3..85f331879a 100644 --- a/services/real-time/app/js/DocumentUpdaterManager.js +++ b/services/real-time/app/js/DocumentUpdaterManager.js @@ -14,7 +14,7 @@ const { UpdateTooLargeError } = require('./Errors') -const rclient = require('redis-sharelatex').createClient( +const rclient = require('@overleaf/redis-wrapper').createClient( settings.redis.documentupdater ) const Keys = settings.redis.documentupdater.key_schema diff --git a/services/real-time/app/js/RedisClientManager.js b/services/real-time/app/js/RedisClientManager.js index d05cc64818..c2070d3ad5 100644 --- a/services/real-time/app/js/RedisClientManager.js +++ b/services/real-time/app/js/RedisClientManager.js @@ -1,4 +1,4 @@ -const redis = require('redis-sharelatex') +const redis = require('@overleaf/redis-wrapper') const logger = require('logger-sharelatex') module.exports = { diff --git a/services/real-time/package-lock.json b/services/real-time/package-lock.json index 56d1b75b2e..925704b791 100644 --- a/services/real-time/package-lock.json +++ b/services/real-time/package-lock.json @@ -607,9 +607,17 @@ } }, "@overleaf/o-error": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/@overleaf/o-error/-/o-error-3.0.0.tgz", - "integrity": "sha512-LsM2s6Iy9G97ktPo0ys4VxtI/m3ahc1ZHwjo5XnhXtjeIkkkVAehsrcRRoV/yWepPjymB0oZonhcfojpjYR/tg==" + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/@overleaf/o-error/-/o-error-3.1.0.tgz", + "integrity": "sha512-TWJ80ozJ1LeugGTJyGQSPEuTkZ9LqZD7/ndLE6azKa03SU/mKV/FINcfk8atpVil8iv1hHQwzYZc35klplpMpQ==" + }, + "@overleaf/redis-wrapper": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/@overleaf/redis-wrapper/-/redis-wrapper-2.0.0.tgz", + "integrity": "sha512-lREuhDPNgmKyOmL1g6onfRzDLWOG/POsE4Vd7ZzLnKDYt9SbOIujtx3CxI2qtQAKBYHf/hfyrbtyX3Ib2yTvYA==", + "requires": { + "ioredis": "~4.17.3" + } }, "@protobufjs/aspromise": { "version": "1.1.2", @@ -2901,11 +2909,11 @@ }, "dependencies": { "debug": { - "version": "4.1.1", - "resolved": "https://registry.npmjs.org/debug/-/debug-4.1.1.tgz", - "integrity": "sha512-pYAIzeRo8J6KPEaJ0VWOh5Pzkbw/RetuzehGM7QRRX5he4fPHx2rdKMB256ehJCkX+XRQm16eZLqLNS8RSZXZw==", + "version": "4.2.0", + "resolved": "https://registry.npmjs.org/debug/-/debug-4.2.0.tgz", + "integrity": "sha512-IX2ncY78vDTjZMFUdmsvIRFY2Cf4FnD0wRs+nQwJU8Lu99/tPFdb0VybiiMTPe3I6rQmwsqQqRBvxU+bZ/I8sg==", "requires": { - "ms": "^2.1.1" + "ms": "2.1.2" } }, "ms": { @@ -3169,7 +3177,8 @@ "lodash": { "version": "4.17.20", "resolved": "https://registry.npmjs.org/lodash/-/lodash-4.17.20.tgz", - "integrity": "sha512-PlhdFcillOINfeV7Ni6oF1TAEayyZBoZ8bcshTHqOYJYlrqzRK5hagpagky5o4HfCzzd1TRkXPMFq6cKk9rGmA==" + "integrity": "sha512-PlhdFcillOINfeV7Ni6oF1TAEayyZBoZ8bcshTHqOYJYlrqzRK5hagpagky5o4HfCzzd1TRkXPMFq6cKk9rGmA==", + "dev": true }, "lodash.at": { "version": "4.6.0", @@ -3184,12 +3193,12 @@ "lodash.defaults": { "version": "4.2.0", "resolved": "https://registry.npmjs.org/lodash.defaults/-/lodash.defaults-4.2.0.tgz", - "integrity": "sha512-qjxPLHd3r5DnsdGacqOMU6pb/avJzdh9tFX2ymgoZE27BmjXrNy/y4LoaiTeAb+O3gL8AfpJGtqfX/ae2leYYQ==" + "integrity": "sha1-0JF4cW/+pN3p5ft7N/bwgCJ0WAw=" }, "lodash.flatten": { "version": "4.4.0", "resolved": "https://registry.npmjs.org/lodash.flatten/-/lodash.flatten-4.4.0.tgz", - "integrity": "sha512-C5N2Z3DgnnKr0LOpv/hKCgKdb7ZZwafIrsesve6lmzvZIRZRGaZ/l6Q8+2W7NaT+ZwO3fFlSCzCzrDCFdJfZ4g==" + "integrity": "sha1-8xwiIlqWMtK7+OSt2+8kCqdlph8=" }, "lodash.has": { "version": "4.5.2", @@ -4657,11 +4666,6 @@ "resolved": "https://registry.npmjs.org/punycode/-/punycode-2.1.1.tgz", "integrity": "sha512-XRsRjdf+j5ml+y/6GKHPZbrF/8p2Yga0JPtdqTIY2Xe5ohJPD9saDJJLPvp9+NSBprVvevdXZybnj2cv8OEd0A==" }, - "q": { - "version": "0.9.2", - "resolved": "https://registry.npmjs.org/q/-/q-0.9.2.tgz", - "integrity": "sha512-ZOxMuWPMJnsUdYhuQ9glpZwKhB4cm8ubYFy1nNCY8TkSAuZun5fd8jCDTlf2ykWnK8x9HGn1stNtLeG179DebQ==" - }, "qs": { "version": "6.7.0", "resolved": "https://registry.npmjs.org/qs/-/qs-6.7.0.tgz", @@ -4767,67 +4771,16 @@ "redis-errors": { "version": "1.2.0", "resolved": "https://registry.npmjs.org/redis-errors/-/redis-errors-1.2.0.tgz", - "integrity": "sha512-1qny3OExCf0UvUV/5wpYKf2YwPcOqXzkwKKSmKHiE6ZMQs5heeE/c8eXK+PNllPvmjgAbfnsbpkGZWy8cBpn9w==" + "integrity": "sha1-62LSrbFeTq9GEMBK/hUpOEJQq60=" }, "redis-parser": { "version": "3.0.0", "resolved": "https://registry.npmjs.org/redis-parser/-/redis-parser-3.0.0.tgz", - "integrity": "sha512-DJnGAeenTdpMEH6uAJRK/uiyEIH9WVsUmoLwzudwGJUwZPp80PDBWPHXSAGNPwNvIXAbe7MSUB1zQFugFml66A==", + "integrity": "sha1-tm2CjNyv5rS4pCin3vTGvKwxyLQ=", "requires": { "redis-errors": "^1.0.0" } }, - "redis-sentinel": { - "version": "0.1.1", - "resolved": "https://registry.npmjs.org/redis-sentinel/-/redis-sentinel-0.1.1.tgz", - "integrity": "sha512-cKtLSUzDsKmsB50J1eIV/SH11DSMiHgsm/gDPRCU5lXz5OyTSuLKWg9oc8d5n74kZwtAyRkfJP0x8vYXvlPjFQ==", - "requires": { - "q": "0.9.2", - "redis": "0.11.x" - }, - "dependencies": { - "redis": { - "version": "0.11.0", - "resolved": "https://registry.npmjs.org/redis/-/redis-0.11.0.tgz", - "integrity": "sha512-wkgzIZ9HuxJ6Sul1IW/6FG13Ecv6q8kmdHb5xo09Hu6bgWzz5qsnM06SVMpDxFNbyApaRjy8CwnmVaRMMhAMWg==" - } - } - }, - "redis-sharelatex": { - "version": "1.0.13", - "resolved": "https://registry.npmjs.org/redis-sharelatex/-/redis-sharelatex-1.0.13.tgz", - "integrity": "sha512-sAQNofqfcMlIxzxNJF1qUspJKDM1VuuIOrGZQX9nb5JtcJ5cusa5sc+Oyb51eymPV5mZGWT3u07tKtv4jdXVIg==", - "requires": { - "async": "^2.5.0", - "coffee-script": "1.8.0", - "ioredis": "~4.17.3", - "redis-sentinel": "0.1.1", - "underscore": "1.7.0" - }, - "dependencies": { - "async": { - "version": "2.6.3", - "resolved": "https://registry.npmjs.org/async/-/async-2.6.3.tgz", - "integrity": "sha512-zflvls11DCy+dQWzTW2dzuilv8Z5X/pjfmZOWba6TNIVDm+2UDaJmXSOXlasHKfNBs8oo3M0aT50fDEWfKZjXg==", - "requires": { - "lodash": "^4.17.14" - } - }, - "coffee-script": { - "version": "1.8.0", - "resolved": "https://registry.npmjs.org/coffee-script/-/coffee-script-1.8.0.tgz", - "integrity": "sha512-EvLTMcu9vR6G1yfnz75yrISvhq1eBPC+pZbQhHzTiC5vXgpYIrArxQc5tB+SYfBi3souVdSZ4AZzYxI72oLXUw==", - "requires": { - "mkdirp": "~0.3.5" - } - }, - "mkdirp": { - "version": "0.3.5", - "resolved": "https://registry.npmjs.org/mkdirp/-/mkdirp-0.3.5.tgz", - "integrity": "sha512-8OCq0De/h9ZxseqzCH8Kw/Filf5pF/vMI6+BH7Lu0jXz2pqYCjTAQRolSxRIi+Ax+oCCjlxoJMP0YQ4XlrQNHg==" - } - } - }, "regexpp": { "version": "2.0.1", "resolved": "https://registry.npmjs.org/regexpp/-/regexpp-2.0.1.tgz", diff --git a/services/real-time/package.json b/services/real-time/package.json index 2eee5a1da1..89012d6ed6 100644 --- a/services/real-time/package.json +++ b/services/real-time/package.json @@ -19,7 +19,8 @@ "format:fix": "node_modules/.bin/prettier-eslint $PWD'/**/*.js' --write" }, "dependencies": { - "@overleaf/o-error": "^3.0.0", + "@overleaf/o-error": "^3.1.0", + "@overleaf/redis-wrapper": "^2.0.0", "async": "^0.9.0", "base64id": "0.1.0", "basic-auth-connect": "^1.0.0", @@ -30,7 +31,6 @@ "express-session": "^1.17.1", "logger-sharelatex": "^2.2.0", "metrics-sharelatex": "^2.6.2", - "redis-sharelatex": "^1.0.13", "request": "^2.88.2", "settings-sharelatex": "^1.1.0", "socket.io": "https://github.com/overleaf/socket.io/archive/0.9.19-overleaf-4.tar.gz", diff --git a/services/real-time/test/acceptance/js/ApplyUpdateTests.js b/services/real-time/test/acceptance/js/ApplyUpdateTests.js index 2c5b753f29..8a750fd68d 100644 --- a/services/real-time/test/acceptance/js/ApplyUpdateTests.js +++ b/services/real-time/test/acceptance/js/ApplyUpdateTests.js @@ -21,7 +21,7 @@ const RealTimeClient = require('./helpers/RealTimeClient') const FixturesManager = require('./helpers/FixturesManager') const settings = require('settings-sharelatex') -const redis = require('redis-sharelatex') +const redis = require('@overleaf/redis-wrapper') const rclient = redis.createClient(settings.redis.documentupdater) const redisSettings = settings.redis diff --git a/services/real-time/test/acceptance/js/EarlyDisconnect.js b/services/real-time/test/acceptance/js/EarlyDisconnect.js index 25e8fbc427..b6d360d0ea 100644 --- a/services/real-time/test/acceptance/js/EarlyDisconnect.js +++ b/services/real-time/test/acceptance/js/EarlyDisconnect.js @@ -19,7 +19,7 @@ const MockWebServer = require('./helpers/MockWebServer') const FixturesManager = require('./helpers/FixturesManager') const settings = require('settings-sharelatex') -const redis = require('redis-sharelatex') +const redis = require('@overleaf/redis-wrapper') const rclient = redis.createClient(settings.redis.pubsub) const rclientRT = redis.createClient(settings.redis.realtime) const KeysRT = settings.redis.realtime.key_schema diff --git a/services/real-time/test/acceptance/js/LeaveProjectTests.js b/services/real-time/test/acceptance/js/LeaveProjectTests.js index 61976d481f..8364e25c4a 100644 --- a/services/real-time/test/acceptance/js/LeaveProjectTests.js +++ b/services/real-time/test/acceptance/js/LeaveProjectTests.js @@ -18,7 +18,7 @@ const FixturesManager = require('./helpers/FixturesManager') const async = require('async') const settings = require('settings-sharelatex') -const redis = require('redis-sharelatex') +const redis = require('@overleaf/redis-wrapper') const rclient = redis.createClient(settings.redis.pubsub) describe('leaveProject', function () { diff --git a/services/real-time/test/acceptance/js/MatrixTests.js b/services/real-time/test/acceptance/js/MatrixTests.js index ff59c4a7ab..2f4b362bca 100644 --- a/services/real-time/test/acceptance/js/MatrixTests.js +++ b/services/real-time/test/acceptance/js/MatrixTests.js @@ -56,7 +56,7 @@ const FixturesManager = require('./helpers/FixturesManager') const settings = require('settings-sharelatex') const Keys = settings.redis.documentupdater.key_schema -const redis = require('redis-sharelatex') +const redis = require('@overleaf/redis-wrapper') const rclient = redis.createClient(settings.redis.pubsub) function getPendingUpdates(doc_id, cb) { diff --git a/services/real-time/test/acceptance/js/PubSubRace.js b/services/real-time/test/acceptance/js/PubSubRace.js index a824ef3e82..a331f73c1b 100644 --- a/services/real-time/test/acceptance/js/PubSubRace.js +++ b/services/real-time/test/acceptance/js/PubSubRace.js @@ -16,7 +16,7 @@ const FixturesManager = require('./helpers/FixturesManager') const async = require('async') const settings = require('settings-sharelatex') -const redis = require('redis-sharelatex') +const redis = require('@overleaf/redis-wrapper') const rclient = redis.createClient(settings.redis.pubsub) describe('PubSubRace', function () { diff --git a/services/real-time/test/acceptance/js/ReceiveUpdateTests.js b/services/real-time/test/acceptance/js/ReceiveUpdateTests.js index 9c65be19f9..75d3991fc3 100644 --- a/services/real-time/test/acceptance/js/ReceiveUpdateTests.js +++ b/services/real-time/test/acceptance/js/ReceiveUpdateTests.js @@ -22,7 +22,7 @@ const FixturesManager = require('./helpers/FixturesManager') const async = require('async') const settings = require('settings-sharelatex') -const redis = require('redis-sharelatex') +const redis = require('@overleaf/redis-wrapper') const rclient = redis.createClient(settings.redis.pubsub) describe('receiveUpdate', function () { diff --git a/services/real-time/test/acceptance/js/helpers/RealTimeClient.js b/services/real-time/test/acceptance/js/helpers/RealTimeClient.js index ab8e222d93..03589e4fa0 100644 --- a/services/real-time/test/acceptance/js/helpers/RealTimeClient.js +++ b/services/real-time/test/acceptance/js/helpers/RealTimeClient.js @@ -18,7 +18,7 @@ const async = require('async') const request = require('request') const Settings = require('settings-sharelatex') -const redis = require('redis-sharelatex') +const redis = require('@overleaf/redis-wrapper') const rclient = redis.createClient(Settings.redis.websessions) const uid = require('uid-safe').sync diff --git a/services/real-time/test/unit/js/ConnectedUsersManagerTests.js b/services/real-time/test/unit/js/ConnectedUsersManagerTests.js index 8e84c41130..6489b20639 100644 --- a/services/real-time/test/unit/js/ConnectedUsersManagerTests.js +++ b/services/real-time/test/unit/js/ConnectedUsersManagerTests.js @@ -59,7 +59,7 @@ describe('ConnectedUsersManager', function () { requires: { 'settings-sharelatex': this.settings, 'logger-sharelatex': { log() {} }, - 'redis-sharelatex': { + '@overleaf/redis-wrapper': { createClient: () => { return this.rClient } diff --git a/services/real-time/test/unit/js/DocumentUpdaterControllerTests.js b/services/real-time/test/unit/js/DocumentUpdaterControllerTests.js index 532346f359..d42505da7e 100644 --- a/services/real-time/test/unit/js/DocumentUpdaterControllerTests.js +++ b/services/real-time/test/unit/js/DocumentUpdaterControllerTests.js @@ -46,7 +46,7 @@ describe('DocumentUpdaterController', function () { pubsub: null } }), - 'redis-sharelatex': (this.redis = { + '@overleaf/redis-wrapper': (this.redis = { createClient: (name) => { let rclientStub this.rclient.push((rclientStub = { name })) diff --git a/services/real-time/test/unit/js/DocumentUpdaterManagerTests.js b/services/real-time/test/unit/js/DocumentUpdaterManagerTests.js index 828f516364..97a032b76f 100644 --- a/services/real-time/test/unit/js/DocumentUpdaterManagerTests.js +++ b/services/real-time/test/unit/js/DocumentUpdaterManagerTests.js @@ -47,7 +47,7 @@ describe('DocumentUpdaterManager', function () { warn: sinon.stub() }), request: (this.request = {}), - 'redis-sharelatex': { createClient: () => this.rclient }, + '@overleaf/redis-wrapper': { createClient: () => this.rclient }, 'metrics-sharelatex': (this.Metrics = { summary: sinon.stub(), Timer: (Timer = class Timer { From 96126ecf34baaeabeba2e1eba6124b9f6d8e4070 Mon Sep 17 00:00:00 2001 From: Eric Mc Sween Date: Mon, 23 Nov 2020 10:56:33 -0500 Subject: [PATCH 453/491] Upgrade build-scripts to 3.4.0 This version fixes docker-compose health checks for dependent services. See https://github.com/overleaf/dev-environment/pull/409 for details. --- services/real-time/buildscript.txt | 2 +- services/real-time/docker-compose.ci.yml | 5 +++++ services/real-time/docker-compose.yml | 5 +++++ 3 files changed, 11 insertions(+), 1 deletion(-) diff --git a/services/real-time/buildscript.txt b/services/real-time/buildscript.txt index 35e4f6c9aa..f75047baec 100644 --- a/services/real-time/buildscript.txt +++ b/services/real-time/buildscript.txt @@ -5,4 +5,4 @@ real-time --env-pass-through= --node-version=10.22.1 --public-repo=True ---script-version=3.3.5 +--script-version=3.4.0 diff --git a/services/real-time/docker-compose.ci.yml b/services/real-time/docker-compose.ci.yml index 255a6b77a6..4e90129f69 100644 --- a/services/real-time/docker-compose.ci.yml +++ b/services/real-time/docker-compose.ci.yml @@ -20,6 +20,7 @@ services: environment: ELASTIC_SEARCH_DSN: es:9200 REDIS_HOST: redis + QUEUES_REDIS_HOST: redis MONGO_HOST: mongo POSTGRES_HOST: postgres MOCHA_GREP: ${MOCHA_GREP} @@ -41,4 +42,8 @@ services: user: root redis: image: redis + healthcheck: + test: ping="$$(redis-cli ping)" && [ "$$ping" = 'PONG' ] + interval: 1s + retries: 20 diff --git a/services/real-time/docker-compose.yml b/services/real-time/docker-compose.yml index 1347882d70..eac74fbee2 100644 --- a/services/real-time/docker-compose.yml +++ b/services/real-time/docker-compose.yml @@ -25,6 +25,7 @@ services: environment: ELASTIC_SEARCH_DSN: es:9200 REDIS_HOST: redis + QUEUES_REDIS_HOST: redis MONGO_HOST: mongo POSTGRES_HOST: postgres MOCHA_GREP: ${MOCHA_GREP} @@ -39,4 +40,8 @@ services: redis: image: redis + healthcheck: + test: ping=$$(redis-cli ping) && [ "$$ping" = 'PONG' ] + interval: 1s + retries: 20 From 746c5aeb801a4623b2f00082ab53be0ab92c6f7b Mon Sep 17 00:00:00 2001 From: Jakob Ackermann Date: Wed, 25 Nov 2020 11:57:22 +0000 Subject: [PATCH 454/491] [misc] bump metrics module to 3.4.1 - renamed package from `metrics-sharelatex` to `@overleaf/metrics` - drop support for statsd backend - decaffeinate - compress `/metrics` response using gzip - bump debugging agents to latest versions - expose prometheus interfaces for custom metrics (custom tags) - cleanup of open sockets metrics - fix deprecation warnings for header access --- services/real-time/app.js | 2 +- services/real-time/app/js/ChannelManager.js | 2 +- .../app/js/DocumentUpdaterController.js | 2 +- .../app/js/DocumentUpdaterManager.js | 2 +- services/real-time/app/js/EventLogger.js | 2 +- .../real-time/app/js/HealthCheckManager.js | 2 +- services/real-time/app/js/RoomManager.js | 2 +- services/real-time/app/js/Router.js | 2 +- .../real-time/app/js/WebsocketController.js | 2 +- services/real-time/package-lock.json | 1416 +++++++++++------ services/real-time/package.json | 2 +- .../test/unit/js/ChannelManagerTests.js | 2 +- .../unit/js/DocumentUpdaterControllerTests.js | 2 +- .../unit/js/DocumentUpdaterManagerTests.js | 2 +- .../test/unit/js/EventLoggerTests.js | 2 +- .../test/unit/js/RoomManagerTests.js | 2 +- .../test/unit/js/WebsocketControllerTests.js | 2 +- 17 files changed, 906 insertions(+), 542 deletions(-) diff --git a/services/real-time/app.js b/services/real-time/app.js index 81c08a722f..7d7d85c83f 100644 --- a/services/real-time/app.js +++ b/services/real-time/app.js @@ -1,4 +1,4 @@ -const Metrics = require('metrics-sharelatex') +const Metrics = require('@overleaf/metrics') const Settings = require('settings-sharelatex') Metrics.initialize(Settings.appName || 'real-time') const async = require('async') diff --git a/services/real-time/app/js/ChannelManager.js b/services/real-time/app/js/ChannelManager.js index 8e2bb57863..1ca9f6e88b 100644 --- a/services/real-time/app/js/ChannelManager.js +++ b/services/real-time/app/js/ChannelManager.js @@ -1,5 +1,5 @@ const logger = require('logger-sharelatex') -const metrics = require('metrics-sharelatex') +const metrics = require('@overleaf/metrics') const settings = require('settings-sharelatex') const OError = require('@overleaf/o-error') diff --git a/services/real-time/app/js/DocumentUpdaterController.js b/services/real-time/app/js/DocumentUpdaterController.js index bf8d25ae85..0e51339600 100644 --- a/services/real-time/app/js/DocumentUpdaterController.js +++ b/services/real-time/app/js/DocumentUpdaterController.js @@ -9,7 +9,7 @@ const EventLogger = require('./EventLogger') const HealthCheckManager = require('./HealthCheckManager') const RoomManager = require('./RoomManager') const ChannelManager = require('./ChannelManager') -const metrics = require('metrics-sharelatex') +const metrics = require('@overleaf/metrics') let DocumentUpdaterController module.exports = DocumentUpdaterController = { diff --git a/services/real-time/app/js/DocumentUpdaterManager.js b/services/real-time/app/js/DocumentUpdaterManager.js index 85f331879a..925d9f38d8 100644 --- a/services/real-time/app/js/DocumentUpdaterManager.js +++ b/services/real-time/app/js/DocumentUpdaterManager.js @@ -6,7 +6,7 @@ const _ = require('underscore') const OError = require('@overleaf/o-error') const logger = require('logger-sharelatex') const settings = require('settings-sharelatex') -const metrics = require('metrics-sharelatex') +const metrics = require('@overleaf/metrics') const { ClientRequestedMissingOpsError, DocumentUpdaterRequestFailedError, diff --git a/services/real-time/app/js/EventLogger.js b/services/real-time/app/js/EventLogger.js index 9db3f8aa55..1a2d898577 100644 --- a/services/real-time/app/js/EventLogger.js +++ b/services/real-time/app/js/EventLogger.js @@ -3,7 +3,7 @@ */ let EventLogger const logger = require('logger-sharelatex') -const metrics = require('metrics-sharelatex') +const metrics = require('@overleaf/metrics') const settings = require('settings-sharelatex') // keep track of message counters to detect duplicate and out of order events diff --git a/services/real-time/app/js/HealthCheckManager.js b/services/real-time/app/js/HealthCheckManager.js index a297807fe8..f521dd9fa0 100644 --- a/services/real-time/app/js/HealthCheckManager.js +++ b/services/real-time/app/js/HealthCheckManager.js @@ -1,4 +1,4 @@ -const metrics = require('metrics-sharelatex') +const metrics = require('@overleaf/metrics') const logger = require('logger-sharelatex') const os = require('os') diff --git a/services/real-time/app/js/RoomManager.js b/services/real-time/app/js/RoomManager.js index d5c0ac7e80..c1e03635a6 100644 --- a/services/real-time/app/js/RoomManager.js +++ b/services/real-time/app/js/RoomManager.js @@ -2,7 +2,7 @@ camelcase, */ const logger = require('logger-sharelatex') -const metrics = require('metrics-sharelatex') +const metrics = require('@overleaf/metrics') const { EventEmitter } = require('events') const OError = require('@overleaf/o-error') diff --git a/services/real-time/app/js/Router.js b/services/real-time/app/js/Router.js index 1aae3ea64e..9b93517ee4 100644 --- a/services/real-time/app/js/Router.js +++ b/services/real-time/app/js/Router.js @@ -1,7 +1,7 @@ /* eslint-disable camelcase, */ -const metrics = require('metrics-sharelatex') +const metrics = require('@overleaf/metrics') const logger = require('logger-sharelatex') const settings = require('settings-sharelatex') const WebsocketController = require('./WebsocketController') diff --git a/services/real-time/app/js/WebsocketController.js b/services/real-time/app/js/WebsocketController.js index 89066a9661..f8e32f0788 100644 --- a/services/real-time/app/js/WebsocketController.js +++ b/services/real-time/app/js/WebsocketController.js @@ -3,7 +3,7 @@ */ const OError = require('@overleaf/o-error') const logger = require('logger-sharelatex') -const metrics = require('metrics-sharelatex') +const metrics = require('@overleaf/metrics') const WebApiManager = require('./WebApiManager') const AuthorizationManager = require('./AuthorizationManager') const DocumentUpdaterManager = require('./DocumentUpdaterManager') diff --git a/services/real-time/package-lock.json b/services/real-time/package-lock.json index 925704b791..0f383213a0 100644 --- a/services/real-time/package-lock.json +++ b/services/real-time/package-lock.json @@ -30,52 +30,6 @@ "js-tokens": "^4.0.0" } }, - "@google-cloud/common": { - "version": "0.32.1", - "resolved": "https://registry.npmjs.org/@google-cloud/common/-/common-0.32.1.tgz", - "integrity": "sha512-bLdPzFvvBMtVkwsoBtygE9oUm3yrNmPa71gvOgucYI/GqvNP2tb6RYsDHPq98kvignhcgHGDI5wyNgxaCo8bKQ==", - "requires": { - "@google-cloud/projectify": "^0.3.3", - "@google-cloud/promisify": "^0.4.0", - "@types/request": "^2.48.1", - "arrify": "^2.0.0", - "duplexify": "^3.6.0", - "ent": "^2.2.0", - "extend": "^3.0.2", - "google-auth-library": "^3.1.1", - "pify": "^4.0.1", - "retry-request": "^4.0.0", - "teeny-request": "^3.11.3" - } - }, - "@google-cloud/debug-agent": { - "version": "3.2.0", - "resolved": "https://registry.npmjs.org/@google-cloud/debug-agent/-/debug-agent-3.2.0.tgz", - "integrity": "sha512-fP87kYbS6aeDna08BivwQ1J260mwJGchRi99XdWCgqbRwuFac8ul0OT5i2wEeDSc5QaDX8ZuWQQ0igZvh1rTyQ==", - "requires": { - "@google-cloud/common": "^0.32.0", - "@sindresorhus/is": "^0.15.0", - "acorn": "^6.0.0", - "coffeescript": "^2.0.0", - "console-log-level": "^1.4.0", - "extend": "^3.0.1", - "findit2": "^2.2.3", - "gcp-metadata": "^1.0.0", - "lodash.pickby": "^4.6.0", - "p-limit": "^2.2.0", - "pify": "^4.0.1", - "semver": "^6.0.0", - "source-map": "^0.6.1", - "split": "^1.0.0" - }, - "dependencies": { - "coffeescript": { - "version": "2.5.1", - "resolved": "https://registry.npmjs.org/coffeescript/-/coffeescript-2.5.1.tgz", - "integrity": "sha512-J2jRPX0eeFh5VKyVnoLrfVFgLZtnnmp96WQSLAS8OrLm2wtQLcnikYKe1gViJKDH7vucjuhHvBKKBP3rKcD1tQ==" - } - } - }, "@google-cloud/logging": { "version": "7.3.0", "resolved": "https://registry.npmjs.org/@google-cloud/logging/-/logging-7.3.0.tgz", @@ -423,136 +377,6 @@ "extend": "^3.0.2" } }, - "@google-cloud/profiler": { - "version": "0.2.3", - "resolved": "https://registry.npmjs.org/@google-cloud/profiler/-/profiler-0.2.3.tgz", - "integrity": "sha512-rNvtrFtIebIxZEJ/O0t8n7HciZGIXBo8DvHxWqAmsCaeLvkTtsaL6HmPkwxrNQ1IhbYWAxF+E/DwCiHyhKmgTg==", - "requires": { - "@google-cloud/common": "^0.26.0", - "@types/console-log-level": "^1.4.0", - "@types/semver": "^5.5.0", - "bindings": "^1.2.1", - "console-log-level": "^1.4.0", - "delay": "^4.0.1", - "extend": "^3.0.1", - "gcp-metadata": "^0.9.0", - "nan": "^2.11.1", - "parse-duration": "^0.1.1", - "pify": "^4.0.0", - "pretty-ms": "^4.0.0", - "protobufjs": "~6.8.6", - "semver": "^5.5.0", - "teeny-request": "^3.3.0" - }, - "dependencies": { - "@google-cloud/common": { - "version": "0.26.2", - "resolved": "https://registry.npmjs.org/@google-cloud/common/-/common-0.26.2.tgz", - "integrity": "sha512-xJ2M/q3MrUbnYZuFlpF01caAlEhAUoRn0NXp93Hn3pkFpfSOG8YfbKbpBAHvcKVbBOAKVIwPsleNtuyuabUwLQ==", - "requires": { - "@google-cloud/projectify": "^0.3.2", - "@google-cloud/promisify": "^0.3.0", - "@types/duplexify": "^3.5.0", - "@types/request": "^2.47.0", - "arrify": "^1.0.1", - "duplexify": "^3.6.0", - "ent": "^2.2.0", - "extend": "^3.0.1", - "google-auth-library": "^2.0.0", - "pify": "^4.0.0", - "retry-request": "^4.0.0", - "through2": "^3.0.0" - } - }, - "@google-cloud/promisify": { - "version": "0.3.1", - "resolved": "https://registry.npmjs.org/@google-cloud/promisify/-/promisify-0.3.1.tgz", - "integrity": "sha512-QzB0/IMvB0eFxFK7Eqh+bfC8NLv3E9ScjWQrPOk6GgfNroxcVITdTlT8NRsRrcp5+QQJVPLkRqKG0PUdaWXmHw==" - }, - "arrify": { - "version": "1.0.1", - "resolved": "https://registry.npmjs.org/arrify/-/arrify-1.0.1.tgz", - "integrity": "sha512-3CYzex9M9FGQjCGMGyi6/31c8GJbgb0qGyrx5HWxPd0aCwh4cB2YjMb2Xf9UuoogrMrlO9cTqnB5rI5GHZTcUA==" - }, - "gcp-metadata": { - "version": "0.9.3", - "resolved": "https://registry.npmjs.org/gcp-metadata/-/gcp-metadata-0.9.3.tgz", - "integrity": "sha512-caV4S84xAjENtpezLCT/GILEAF5h/bC4cNqZFmt/tjTn8t+JBtTkQrgBrJu3857YdsnlM8rxX/PMcKGtE8hUlw==", - "requires": { - "gaxios": "^1.0.2", - "json-bigint": "^0.3.0" - } - }, - "google-auth-library": { - "version": "2.0.2", - "resolved": "https://registry.npmjs.org/google-auth-library/-/google-auth-library-2.0.2.tgz", - "integrity": "sha512-FURxmo1hBVmcfLauuMRKOPYAPKht3dGuI2wjeJFalDUThO0HoYVjr4yxt5cgYSFm1dgUpmN9G/poa7ceTFAIiA==", - "requires": { - "axios": "^0.18.0", - "gcp-metadata": "^0.7.0", - "gtoken": "^2.3.0", - "https-proxy-agent": "^2.2.1", - "jws": "^3.1.5", - "lru-cache": "^5.0.0", - "semver": "^5.5.0" - }, - "dependencies": { - "gcp-metadata": { - "version": "0.7.0", - "resolved": "https://registry.npmjs.org/gcp-metadata/-/gcp-metadata-0.7.0.tgz", - "integrity": "sha512-ffjC09amcDWjh3VZdkDngIo7WoluyC5Ag9PAYxZbmQLOLNI8lvPtoKTSCyU54j2gwy5roZh6sSMTfkY2ct7K3g==", - "requires": { - "axios": "^0.18.0", - "extend": "^3.0.1", - "retry-axios": "0.3.2" - } - } - } - }, - "semver": { - "version": "5.7.1", - "resolved": "https://registry.npmjs.org/semver/-/semver-5.7.1.tgz", - "integrity": "sha512-sauaDf/PZdVgrLTNYHRtpXa1iRiKcaebiKQ1BJdpQlWH2lCvexQdX55snPFyK7QzpudqbCI0qXFfOasHdyNDGQ==" - } - } - }, - "@google-cloud/projectify": { - "version": "0.3.3", - "resolved": "https://registry.npmjs.org/@google-cloud/projectify/-/projectify-0.3.3.tgz", - "integrity": "sha512-7522YHQ4IhaafgSunsFF15nG0TGVmxgXidy9cITMe+256RgqfcrfWphiMufW+Ou4kqagW/u3yxwbzVEW3dk2Uw==" - }, - "@google-cloud/promisify": { - "version": "0.4.0", - "resolved": "https://registry.npmjs.org/@google-cloud/promisify/-/promisify-0.4.0.tgz", - "integrity": "sha512-4yAHDC52TEMCNcMzVC8WlqnKKKq+Ssi2lXoUg9zWWkZ6U6tq9ZBRYLHHCRdfU+EU9YJsVmivwGcKYCjRGjnf4Q==" - }, - "@google-cloud/trace-agent": { - "version": "3.6.1", - "resolved": "https://registry.npmjs.org/@google-cloud/trace-agent/-/trace-agent-3.6.1.tgz", - "integrity": "sha512-KDo85aPN4gSxJ7oEIOlKd7aGENZFXAM1kbIn1Ds+61gh/K1CQWSyepgJo3nUpAwH6D1ezDWV7Iaf8ueoITc8Uw==", - "requires": { - "@google-cloud/common": "^0.32.1", - "builtin-modules": "^3.0.0", - "console-log-level": "^1.4.0", - "continuation-local-storage": "^3.2.1", - "extend": "^3.0.0", - "gcp-metadata": "^1.0.0", - "hex2dec": "^1.0.1", - "is": "^3.2.0", - "methods": "^1.1.1", - "require-in-the-middle": "^4.0.0", - "semver": "^6.0.0", - "shimmer": "^1.2.0", - "uuid": "^3.0.1" - }, - "dependencies": { - "uuid": { - "version": "3.4.0", - "resolved": "https://registry.npmjs.org/uuid/-/uuid-3.4.0.tgz", - "integrity": "sha512-HjSDRw6gZE5JMggctHBcjVak08+KEVhSIiDzFnT9S9aegmp85S/bReBVTb4QTFaRNptJ9kuYaNhnbNEOkbKb/A==" - } - } - }, "@grpc/grpc-js": { "version": "1.0.5", "resolved": "https://registry.npmjs.org/@grpc/grpc-js/-/grpc-js-1.0.5.tgz", @@ -606,6 +430,386 @@ } } }, + "@overleaf/metrics": { + "version": "3.4.1", + "resolved": "https://registry.npmjs.org/@overleaf/metrics/-/metrics-3.4.1.tgz", + "integrity": "sha512-OgjlzuC+2gPdIEDHhmd9LDMu01tk1ln0cJhw1727BZ+Wgf2Z1hjuHRt4JeCkf+PFTHwJutVYT8v6IGPpNEPtbg==", + "requires": { + "@google-cloud/debug-agent": "^5.1.2", + "@google-cloud/profiler": "^4.0.3", + "@google-cloud/trace-agent": "^5.1.1", + "compression": "^1.7.4", + "prom-client": "^11.1.3", + "underscore": "~1.6.0", + "yn": "^3.1.1" + }, + "dependencies": { + "@google-cloud/common": { + "version": "3.5.0", + "resolved": "https://registry.npmjs.org/@google-cloud/common/-/common-3.5.0.tgz", + "integrity": "sha512-10d7ZAvKhq47L271AqvHEd8KzJqGU45TY+rwM2Z3JHuB070FeTi7oJJd7elfrnKaEvaktw3hH2wKnRWxk/3oWQ==", + "requires": { + "@google-cloud/projectify": "^2.0.0", + "@google-cloud/promisify": "^2.0.0", + "arrify": "^2.0.1", + "duplexify": "^4.1.1", + "ent": "^2.2.0", + "extend": "^3.0.2", + "google-auth-library": "^6.1.1", + "retry-request": "^4.1.1", + "teeny-request": "^7.0.0" + } + }, + "@google-cloud/debug-agent": { + "version": "5.1.3", + "resolved": "https://registry.npmjs.org/@google-cloud/debug-agent/-/debug-agent-5.1.3.tgz", + "integrity": "sha512-WbzeEz4MvPlM7DX2QBsPcWgF62u7LSQv/oMYPl0L+TddTebqjDKiVXwxpzWk61NIfcKiet3dyCbPIt3N5o8XPQ==", + "requires": { + "@google-cloud/common": "^3.0.0", + "acorn": "^8.0.0", + "coffeescript": "^2.0.0", + "console-log-level": "^1.4.0", + "extend": "^3.0.2", + "findit2": "^2.2.3", + "gcp-metadata": "^4.0.0", + "p-limit": "^3.0.1", + "semver": "^7.0.0", + "source-map": "^0.6.1", + "split": "^1.0.0" + } + }, + "@google-cloud/profiler": { + "version": "4.1.0", + "resolved": "https://registry.npmjs.org/@google-cloud/profiler/-/profiler-4.1.0.tgz", + "integrity": "sha512-9e1zXRctLSUHAoAsFGwE4rS28fr0siiG+jXl5OpwTK8ZAUlxb70aosHaZGdsv8YXrYKjuiufjRZ/OXCs0XLI9g==", + "requires": { + "@google-cloud/common": "^3.0.0", + "@types/console-log-level": "^1.4.0", + "@types/semver": "^7.0.0", + "console-log-level": "^1.4.0", + "delay": "^4.0.1", + "extend": "^3.0.2", + "gcp-metadata": "^4.0.0", + "parse-duration": "^0.4.4", + "pprof": "3.0.0", + "pretty-ms": "^7.0.0", + "protobufjs": "~6.10.0", + "semver": "^7.0.0", + "teeny-request": "^7.0.0" + } + }, + "@google-cloud/projectify": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/@google-cloud/projectify/-/projectify-2.0.1.tgz", + "integrity": "sha512-ZDG38U/Yy6Zr21LaR3BTiiLtpJl6RkPS/JwoRT453G+6Q1DhlV0waNf8Lfu+YVYGIIxgKnLayJRfYlFJfiI8iQ==" + }, + "@google-cloud/promisify": { + "version": "2.0.3", + "resolved": "https://registry.npmjs.org/@google-cloud/promisify/-/promisify-2.0.3.tgz", + "integrity": "sha512-d4VSA86eL/AFTe5xtyZX+ePUjE8dIFu2T8zmdeNBSa5/kNgXPCx/o/wbFNHAGLJdGnk1vddRuMESD9HbOC8irw==" + }, + "@google-cloud/trace-agent": { + "version": "5.1.1", + "resolved": "https://registry.npmjs.org/@google-cloud/trace-agent/-/trace-agent-5.1.1.tgz", + "integrity": "sha512-YTcK0RLN90pLCprg0XC8uV4oAVd79vsXhkcxmEVwiOOYjUDvSrAhb7y/0SY606zgfhJHmUTNb/fZSWEtZP/slQ==", + "requires": { + "@google-cloud/common": "^3.0.0", + "@opencensus/propagation-stackdriver": "0.0.22", + "builtin-modules": "^3.0.0", + "console-log-level": "^1.4.0", + "continuation-local-storage": "^3.2.1", + "extend": "^3.0.2", + "gcp-metadata": "^4.0.0", + "google-auth-library": "^6.0.0", + "hex2dec": "^1.0.1", + "is": "^3.2.0", + "methods": "^1.1.1", + "require-in-the-middle": "^5.0.0", + "semver": "^7.0.0", + "shimmer": "^1.2.0", + "source-map-support": "^0.5.16", + "uuid": "^8.0.0" + } + }, + "@opencensus/core": { + "version": "0.0.22", + "resolved": "https://registry.npmjs.org/@opencensus/core/-/core-0.0.22.tgz", + "integrity": "sha512-ErazJtivjceNoOZI1bG9giQ6cWS45J4i6iPUtlp7dLNu58OLs/v+CD0FsaPCh47XgPxAI12vbBE8Ec09ViwHNA==", + "requires": { + "continuation-local-storage": "^3.2.1", + "log-driver": "^1.2.7", + "semver": "^7.0.0", + "shimmer": "^1.2.0", + "uuid": "^8.0.0" + } + }, + "@opencensus/propagation-stackdriver": { + "version": "0.0.22", + "resolved": "https://registry.npmjs.org/@opencensus/propagation-stackdriver/-/propagation-stackdriver-0.0.22.tgz", + "integrity": "sha512-eBvf/ihb1mN8Yz/ASkz8nHzuMKqygu77+VNnUeR0yEh3Nj+ykB8VVR6lK+NAFXo1Rd1cOsTmgvuXAZgDAGleQQ==", + "requires": { + "@opencensus/core": "^0.0.22", + "hex2dec": "^1.0.1", + "uuid": "^8.0.0" + } + }, + "@types/semver": { + "version": "7.3.4", + "resolved": "https://registry.npmjs.org/@types/semver/-/semver-7.3.4.tgz", + "integrity": "sha512-+nVsLKlcUCeMzD2ufHEYuJ9a2ovstb6Dp52A5VsoKxDXgvE051XgHI/33I1EymwkRGQkwnA0LkhnUzituGs4EQ==" + }, + "acorn": { + "version": "8.0.4", + "resolved": "https://registry.npmjs.org/acorn/-/acorn-8.0.4.tgz", + "integrity": "sha512-XNP0PqF1XD19ZlLKvB7cMmnZswW4C/03pRHgirB30uSJTaS3A3V1/P4sS3HPvFmjoriPCJQs+JDSbm4bL1TxGQ==" + }, + "agent-base": { + "version": "6.0.2", + "resolved": "https://registry.npmjs.org/agent-base/-/agent-base-6.0.2.tgz", + "integrity": "sha512-RZNwNclF7+MS/8bDg70amg32dyeZGZxiDuQmZxKLAlQjr3jGyLx+4Kkk58UO7D2QdgFIQCovuSuZESne6RG6XQ==", + "requires": { + "debug": "4" + } + }, + "bignumber.js": { + "version": "9.0.1", + "resolved": "https://registry.npmjs.org/bignumber.js/-/bignumber.js-9.0.1.tgz", + "integrity": "sha512-IdZR9mh6ahOBv/hYGiXyVuyCetmGJhtYkqLBpTStdhEGjegpPlUawydyaF3pbIOFynJTpllEs+NP+CS9jKFLjA==" + }, + "coffeescript": { + "version": "2.5.1", + "resolved": "https://registry.npmjs.org/coffeescript/-/coffeescript-2.5.1.tgz", + "integrity": "sha512-J2jRPX0eeFh5VKyVnoLrfVFgLZtnnmp96WQSLAS8OrLm2wtQLcnikYKe1gViJKDH7vucjuhHvBKKBP3rKcD1tQ==" + }, + "debug": { + "version": "4.3.1", + "resolved": "https://registry.npmjs.org/debug/-/debug-4.3.1.tgz", + "integrity": "sha512-doEwdvm4PCeK4K3RQN2ZC2BYUBaxwLARCqZmMjtF8a51J2Rb0xpVloFRnCODwqjpwnAoao4pelN8l3RJdv3gRQ==", + "requires": { + "ms": "2.1.2" + } + }, + "duplexify": { + "version": "4.1.1", + "resolved": "https://registry.npmjs.org/duplexify/-/duplexify-4.1.1.tgz", + "integrity": "sha512-DY3xVEmVHTv1wSzKNbwoU6nVjzI369Y6sPoqfYr0/xlx3IdX2n94xIszTcjPO8W8ZIv0Wb0PXNcjuZyT4wiICA==", + "requires": { + "end-of-stream": "^1.4.1", + "inherits": "^2.0.3", + "readable-stream": "^3.1.1", + "stream-shift": "^1.0.0" + } + }, + "gaxios": { + "version": "4.0.1", + "resolved": "https://registry.npmjs.org/gaxios/-/gaxios-4.0.1.tgz", + "integrity": "sha512-jOin8xRZ/UytQeBpSXFqIzqU7Fi5TqgPNLlUsSB8kjJ76+FiGBfImF8KJu++c6J4jOldfJUtt0YmkRj2ZpSHTQ==", + "requires": { + "abort-controller": "^3.0.0", + "extend": "^3.0.2", + "https-proxy-agent": "^5.0.0", + "is-stream": "^2.0.0", + "node-fetch": "^2.3.0" + } + }, + "gcp-metadata": { + "version": "4.2.1", + "resolved": "https://registry.npmjs.org/gcp-metadata/-/gcp-metadata-4.2.1.tgz", + "integrity": "sha512-tSk+REe5iq/N+K+SK1XjZJUrFPuDqGZVzCy2vocIHIGmPlTGsa8owXMJwGkrXr73NO0AzhPW4MF2DEHz7P2AVw==", + "requires": { + "gaxios": "^4.0.0", + "json-bigint": "^1.0.0" + } + }, + "google-auth-library": { + "version": "6.1.3", + "resolved": "https://registry.npmjs.org/google-auth-library/-/google-auth-library-6.1.3.tgz", + "integrity": "sha512-m9mwvY3GWbr7ZYEbl61isWmk+fvTmOt0YNUfPOUY2VH8K5pZlAIWJjxEi0PqR3OjMretyiQLI6GURMrPSwHQ2g==", + "requires": { + "arrify": "^2.0.0", + "base64-js": "^1.3.0", + "ecdsa-sig-formatter": "^1.0.11", + "fast-text-encoding": "^1.0.0", + "gaxios": "^4.0.0", + "gcp-metadata": "^4.2.0", + "gtoken": "^5.0.4", + "jws": "^4.0.0", + "lru-cache": "^6.0.0" + } + }, + "google-p12-pem": { + "version": "3.0.3", + "resolved": "https://registry.npmjs.org/google-p12-pem/-/google-p12-pem-3.0.3.tgz", + "integrity": "sha512-wS0ek4ZtFx/ACKYF3JhyGe5kzH7pgiQ7J5otlumqR9psmWMYc+U9cErKlCYVYHoUaidXHdZ2xbo34kB+S+24hA==", + "requires": { + "node-forge": "^0.10.0" + } + }, + "gtoken": { + "version": "5.1.0", + "resolved": "https://registry.npmjs.org/gtoken/-/gtoken-5.1.0.tgz", + "integrity": "sha512-4d8N6Lk8TEAHl9vVoRVMh9BNOKWVgl2DdNtr3428O75r3QFrF/a5MMu851VmK0AA8+iSvbwRv69k5XnMLURGhg==", + "requires": { + "gaxios": "^4.0.0", + "google-p12-pem": "^3.0.3", + "jws": "^4.0.0", + "mime": "^2.2.0" + } + }, + "https-proxy-agent": { + "version": "5.0.0", + "resolved": "https://registry.npmjs.org/https-proxy-agent/-/https-proxy-agent-5.0.0.tgz", + "integrity": "sha512-EkYm5BcKUGiduxzSt3Eppko+PiNWNEpa4ySk9vTC6wDsQJW9rHSa+UhGNJoRYp7bz6Ht1eaRIa6QaJqO5rCFbA==", + "requires": { + "agent-base": "6", + "debug": "4" + } + }, + "json-bigint": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/json-bigint/-/json-bigint-1.0.0.tgz", + "integrity": "sha512-SiPv/8VpZuWbvLSMtTDU8hEfrZWg/mH/nV/b4o0CYbSxu1UIQPLdwKOCIyLQX+VIPO5vrLX3i8qtqFyhdPSUSQ==", + "requires": { + "bignumber.js": "^9.0.0" + } + }, + "jwa": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/jwa/-/jwa-2.0.0.tgz", + "integrity": "sha512-jrZ2Qx916EA+fq9cEAeCROWPTfCwi1IVHqT2tapuqLEVVDKFDENFw1oL+MwrTvH6msKxsd1YTDVw6uKEcsrLEA==", + "requires": { + "buffer-equal-constant-time": "1.0.1", + "ecdsa-sig-formatter": "1.0.11", + "safe-buffer": "^5.0.1" + } + }, + "jws": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/jws/-/jws-4.0.0.tgz", + "integrity": "sha512-KDncfTmOZoOMTFG4mBlG0qUIOlc03fmzH+ru6RgYVZhPkyiy/92Owlt/8UEN+a4TXR1FQetfIpJE8ApdvdVxTg==", + "requires": { + "jwa": "^2.0.0", + "safe-buffer": "^5.0.1" + } + }, + "lru-cache": { + "version": "6.0.0", + "resolved": "https://registry.npmjs.org/lru-cache/-/lru-cache-6.0.0.tgz", + "integrity": "sha512-Jo6dJ04CmSjuznwJSS3pUeWmd/H0ffTlkXXgwZi+eq1UCmqQwCh+eLsYOYCwY991i2Fah4h1BEMCx4qThGbsiA==", + "requires": { + "yallist": "^4.0.0" + } + }, + "mime": { + "version": "2.4.6", + "resolved": "https://registry.npmjs.org/mime/-/mime-2.4.6.tgz", + "integrity": "sha512-RZKhC3EmpBchfTGBVb8fb+RL2cWyw/32lshnsETttkBAyAUXSGHxbEJWWRXc751DrIxG1q04b8QwMbAwkRPpUA==" + }, + "ms": { + "version": "2.1.2", + "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.2.tgz", + "integrity": "sha512-sGkPx+VjMtmA6MX27oA4FBFELFCZZ4S4XqeGOXCv68tT+jb3vk/RyaKWP0PTKyWtmLSM0b+adUTEvbs1PEaH2w==" + }, + "node-forge": { + "version": "0.10.0", + "resolved": "https://registry.npmjs.org/node-forge/-/node-forge-0.10.0.tgz", + "integrity": "sha512-PPmu8eEeG9saEUvI97fm4OYxXVB6bFvyNTyiUOBichBpFG8A1Ljw3bY62+5oOjDEMHRnd0Y7HQ+x7uzxOzC6JA==" + }, + "p-limit": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/p-limit/-/p-limit-3.1.0.tgz", + "integrity": "sha512-TYOanM3wGwNGsZN2cVTYPArw454xnXj5qmWF1bEoAc4+cU/ol7GVh7odevjp1FNHduHc3KZMcFduxU5Xc6uJRQ==", + "requires": { + "yocto-queue": "^0.1.0" + } + }, + "parse-duration": { + "version": "0.4.4", + "resolved": "https://registry.npmjs.org/parse-duration/-/parse-duration-0.4.4.tgz", + "integrity": "sha512-KbAJuYGUhZkB9gotDiKLnZ7Z3VTacK3fgwmDdB6ZVDtJbMBT6MfLga0WJaYpPDu0mzqT0NgHtHDt5PY4l0nidg==" + }, + "pretty-ms": { + "version": "7.0.1", + "resolved": "https://registry.npmjs.org/pretty-ms/-/pretty-ms-7.0.1.tgz", + "integrity": "sha512-973driJZvxiGOQ5ONsFhOF/DtzPMOMtgC11kCpUrPGMTgqp2q/1gwzCquocrN33is0VZ5GFHXZYMM9l6h67v2Q==", + "requires": { + "parse-ms": "^2.1.0" + } + }, + "protobufjs": { + "version": "6.10.2", + "resolved": "https://registry.npmjs.org/protobufjs/-/protobufjs-6.10.2.tgz", + "integrity": "sha512-27yj+04uF6ya9l+qfpH187aqEzfCF4+Uit0I9ZBQVqK09hk/SQzKa2MUqUpXaVa7LOFRg1TSSr3lVxGOk6c0SQ==", + "requires": { + "@protobufjs/aspromise": "^1.1.2", + "@protobufjs/base64": "^1.1.2", + "@protobufjs/codegen": "^2.0.4", + "@protobufjs/eventemitter": "^1.1.0", + "@protobufjs/fetch": "^1.1.0", + "@protobufjs/float": "^1.0.2", + "@protobufjs/inquire": "^1.1.0", + "@protobufjs/path": "^1.1.2", + "@protobufjs/pool": "^1.1.0", + "@protobufjs/utf8": "^1.1.0", + "@types/long": "^4.0.1", + "@types/node": "^13.7.0", + "long": "^4.0.0" + } + }, + "readable-stream": { + "version": "3.6.0", + "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-3.6.0.tgz", + "integrity": "sha512-BViHy7LKeTz4oNnkcLJ+lVSL6vpiFeX6/d3oSH8zCW7UxP2onchk+vTGB143xuFjHS3deTgkKoXXymXqymiIdA==", + "requires": { + "inherits": "^2.0.3", + "string_decoder": "^1.1.1", + "util-deprecate": "^1.0.1" + } + }, + "require-in-the-middle": { + "version": "5.0.3", + "resolved": "https://registry.npmjs.org/require-in-the-middle/-/require-in-the-middle-5.0.3.tgz", + "integrity": "sha512-p/ICV8uMlqC4tjOYabLMxAWCIKa0YUQgZZ6KDM0xgXJNgdGQ1WmL2A07TwmrZw+wi6ITUFKzH5v3n+ENEyXVkA==", + "requires": { + "debug": "^4.1.1", + "module-details-from-path": "^1.0.3", + "resolve": "^1.12.0" + } + }, + "semver": { + "version": "7.3.2", + "resolved": "https://registry.npmjs.org/semver/-/semver-7.3.2.tgz", + "integrity": "sha512-OrOb32TeeambH6UrhtShmF7CRDqhL6/5XpPNp2DuRH6+9QLw/orhp72j87v8Qa1ScDkvrrBNpZcDejAirJmfXQ==" + }, + "teeny-request": { + "version": "7.0.1", + "resolved": "https://registry.npmjs.org/teeny-request/-/teeny-request-7.0.1.tgz", + "integrity": "sha512-sasJmQ37klOlplL4Ia/786M5YlOcoLGQyq2TE4WHSRupbAuDaQW0PfVxV4MtdBtRJ4ngzS+1qim8zP6Zp35qCw==", + "requires": { + "http-proxy-agent": "^4.0.0", + "https-proxy-agent": "^5.0.0", + "node-fetch": "^2.6.1", + "stream-events": "^1.0.5", + "uuid": "^8.0.0" + } + }, + "underscore": { + "version": "1.6.0", + "resolved": "https://registry.npmjs.org/underscore/-/underscore-1.6.0.tgz", + "integrity": "sha1-izixDKze9jM3uLJOT/htRa6lKag=" + }, + "uuid": { + "version": "8.3.1", + "resolved": "https://registry.npmjs.org/uuid/-/uuid-8.3.1.tgz", + "integrity": "sha512-FOmRr+FmWEIG8uhZv6C2bTgEVXsHk08kE7mPlrBbEe+c3r9pjceVPgupIfNIhc4yx55H69OXANrUaSuu9eInKg==" + }, + "yallist": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/yallist/-/yallist-4.0.0.tgz", + "integrity": "sha512-3wdGidZyq5PB084XLES5TpOSRA3wjXAlIWMhum2kRcv/41Sn2emQ0dycQW4uZXLejwKvg6EsvbdlVL+FYEct7A==" + } + } + }, "@overleaf/o-error": { "version": "3.1.0", "resolved": "https://registry.npmjs.org/@overleaf/o-error/-/o-error-3.1.0.tgz", @@ -673,21 +877,11 @@ "resolved": "https://registry.npmjs.org/@protobufjs/utf8/-/utf8-1.1.0.tgz", "integrity": "sha512-Vvn3zZrhQZkkBE8LSuW3em98c0FwgO4nxzv6OdSxPKJIEKY2bGbHn+mhGIPerzI4twdxaP8/0+06HBpwf345Lw==" }, - "@sindresorhus/is": { - "version": "0.15.0", - "resolved": "https://registry.npmjs.org/@sindresorhus/is/-/is-0.15.0.tgz", - "integrity": "sha512-lu8BpxjAtRCAo5ifytTpCPCj99LF7o/2Myn+NXyNCBqvPYn7Pjd76AMmUB5l7XF1U6t0hcWrlEM5ESufW7wAeA==" - }, "@tootallnate/once": { "version": "1.1.2", "resolved": "https://registry.npmjs.org/@tootallnate/once/-/once-1.1.2.tgz", "integrity": "sha512-RbzJvlNzmRq5c3O09UipeuXno4tA1FE6ikOjxZK0tuxVv3412l64l5t1W5pj4+rJq9vpkm/kwiR07aZXnsKPxw==" }, - "@types/caseless": { - "version": "0.12.2", - "resolved": "https://registry.npmjs.org/@types/caseless/-/caseless-0.12.2.tgz", - "integrity": "sha512-6ckxMjBBD8URvjB6J3NcnuAn5Pkl7t3TizAg+xdlzzQGSPSmBcXf8KoIH0ua/i+tio+ZRUHEXp0HEmvaR4kt0w==" - }, "@types/color-name": { "version": "1.1.1", "resolved": "https://registry.npmjs.org/@types/color-name/-/color-name-1.1.1.tgz", @@ -699,14 +893,6 @@ "resolved": "https://registry.npmjs.org/@types/console-log-level/-/console-log-level-1.4.0.tgz", "integrity": "sha512-x+OscEQwcx5Biair4enH7ov9W+clcqUWaZRaxn5IkT4yNWWjRr2oiYDkY/x1uXSTVZOQ2xlbFQySaQGB+VdXGQ==" }, - "@types/duplexify": { - "version": "3.6.0", - "resolved": "https://registry.npmjs.org/@types/duplexify/-/duplexify-3.6.0.tgz", - "integrity": "sha512-5zOA53RUlzN74bvrSGwjudssD9F3a797sDZQkiYpUOxW+WHaXTCPz4/d5Dgi6FKnOqZ2CpaTo0DhgIfsXAOE/A==", - "requires": { - "@types/node": "*" - } - }, "@types/eslint-visitor-keys": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/@types/eslint-visitor-keys/-/eslint-visitor-keys-1.0.0.tgz", @@ -743,39 +929,6 @@ "resolved": "https://registry.npmjs.org/@types/node/-/node-13.9.5.tgz", "integrity": "sha512-hkzMMD3xu6BrJpGVLeQ3htQQNAcOrJjX7WFmtK8zWQpz2UJf13LCFF2ALA7c9OVdvc2vQJeDdjfR35M0sBCxvw==" }, - "@types/request": { - "version": "2.48.4", - "resolved": "https://registry.npmjs.org/@types/request/-/request-2.48.4.tgz", - "integrity": "sha512-W1t1MTKYR8PxICH+A4HgEIPuAC3sbljoEVfyZbeFJJDbr30guDspJri2XOaM2E+Un7ZjrihaDi7cf6fPa2tbgw==", - "requires": { - "@types/caseless": "*", - "@types/node": "*", - "@types/tough-cookie": "*", - "form-data": "^2.5.0" - }, - "dependencies": { - "form-data": { - "version": "2.5.1", - "resolved": "https://registry.npmjs.org/form-data/-/form-data-2.5.1.tgz", - "integrity": "sha512-m21N3WOmEEURgk6B9GLOE4RuWOFf28Lhh9qGYeNlGq4VDXUlJy2th2slBNU8Gp8EzloYZOibZJ7t5ecIrFSjVA==", - "requires": { - "asynckit": "^0.4.0", - "combined-stream": "^1.0.6", - "mime-types": "^2.1.12" - } - } - } - }, - "@types/semver": { - "version": "5.5.0", - "resolved": "https://registry.npmjs.org/@types/semver/-/semver-5.5.0.tgz", - "integrity": "sha512-41qEJgBH/TWgo5NFSvBCJ1qkoi3Q6ONSF2avrHq1LVEZfYpdHmj0y9SuTK+u9ZhG1sYQKBL1AWXKyLWP4RaUoQ==" - }, - "@types/tough-cookie": { - "version": "2.3.6", - "resolved": "https://registry.npmjs.org/@types/tough-cookie/-/tough-cookie-2.3.6.tgz", - "integrity": "sha512-wHNBMnkoEBiRAd3s8KTKwIuO9biFtTf0LehITzBhSco+HQI0xkXZbLOD55SW3Aqw3oUkHstkm5SPv58yaAdFPQ==" - }, "@typescript-eslint/experimental-utils": { "version": "1.13.0", "resolved": "https://registry.npmjs.org/@typescript-eslint/experimental-utils/-/experimental-utils-1.13.0.tgz", @@ -829,6 +982,11 @@ } } }, + "abbrev": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/abbrev/-/abbrev-1.1.1.tgz", + "integrity": "sha512-nne9/IiQ/hzIhY6pdDnbBtz7DjPTKrY00P/zvPSm5pOFkl6xuGrGnXn/VtTNNfNtAfZ9/1RtehkszU9qcTii0Q==" + }, "abort-controller": { "version": "3.0.0", "resolved": "https://registry.npmjs.org/abort-controller/-/abort-controller-3.0.0.tgz", @@ -864,7 +1022,8 @@ "acorn": { "version": "6.4.1", "resolved": "https://registry.npmjs.org/acorn/-/acorn-6.4.1.tgz", - "integrity": "sha512-ZVA9k326Nwrj3Cj9jlh3wGFutC2ZornPNARZwsNYqQYgN0EsV2d53w5RN/co65Ohn4sUAUtb1rSUAOD6XN9idA==" + "integrity": "sha512-ZVA9k326Nwrj3Cj9jlh3wGFutC2ZornPNARZwsNYqQYgN0EsV2d53w5RN/co65Ohn4sUAUtb1rSUAOD6XN9idA==", + "dev": true }, "acorn-jsx": { "version": "5.2.0", @@ -880,14 +1039,6 @@ "zeparser": "0.0.5" } }, - "agent-base": { - "version": "4.3.0", - "resolved": "https://registry.npmjs.org/agent-base/-/agent-base-4.3.0.tgz", - "integrity": "sha512-salcGninV0nPrwpGNn4VTXBb1SOuXQBiqbrNXoeizJsHrsL6ERFM2Ne3JUSBWRE6aeNJI2ROP/WEEIDUiDe3cg==", - "requires": { - "es6-promisify": "^5.0.0" - } - }, "ajv": { "version": "6.12.2", "resolved": "https://registry.npmjs.org/ajv/-/ajv-6.12.2.tgz", @@ -931,6 +1082,20 @@ "color-convert": "^1.9.0" } }, + "aproba": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/aproba/-/aproba-1.2.0.tgz", + "integrity": "sha512-Y9J6ZjXtoYh8RnXVCMOU/ttDmk1aBjunq9vO0ta5x85WDQiQfUF9sIPBITdbiiIVcBo03Hi3jMxigBtsddlXRw==" + }, + "are-we-there-yet": { + "version": "1.1.5", + "resolved": "https://registry.npmjs.org/are-we-there-yet/-/are-we-there-yet-1.1.5.tgz", + "integrity": "sha512-5hYdAkZlcG8tOLujVDTgCT+uPX0VnpAH28gWsLfzpXYm7wP6mp5Q/gYyR7YQ0cKVJcXJnl3j2kpBan13PtQf6w==", + "requires": { + "delegates": "^1.0.0", + "readable-stream": "^2.0.6" + } + }, "argparse": { "version": "1.0.10", "resolved": "https://registry.npmjs.org/argparse/-/argparse-1.0.10.tgz", @@ -1032,15 +1197,6 @@ "resolved": "https://registry.npmjs.org/aws4/-/aws4-1.10.0.tgz", "integrity": "sha512-3YDiu347mtVtjpyV3u5kVqQLP242c06zwDOgpeRnybmXlYYsLbtTrUBUm8i8srONt+FWobl5aibnU1030PeeuA==" }, - "axios": { - "version": "0.18.1", - "resolved": "https://registry.npmjs.org/axios/-/axios-0.18.1.tgz", - "integrity": "sha512-0BfJq4NSfQXd+SkFdrvFbG7addhYSBA2mQwISr46pD6E5iqkWg02RAs8vyTT/j0RTnoYmeXauBuSv1qKwR179g==", - "requires": { - "follow-redirects": "1.5.10", - "is-buffer": "^2.0.2" - } - }, "balanced-match": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/balanced-match/-/balanced-match-1.0.0.tgz", @@ -1130,6 +1286,11 @@ "resolved": "https://registry.npmjs.org/buffer-equal-constant-time/-/buffer-equal-constant-time-1.0.1.tgz", "integrity": "sha512-zRpUiDwd/xk6ADqPMATG8vc9VPrkck7T07OIx0gnjmJAnHnTVXNQG3vfvWNuiZIkwu9KrKdA1iJKfsfTVxE6NA==" }, + "buffer-from": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/buffer-from/-/buffer-from-1.1.1.tgz", + "integrity": "sha512-MQcXEUbCKtEo7bhqEs6560Hyd4XaovZlO/k9V3hjVUF/zwW7KBVdSK4gIt/bzwS9MbR5qob+F5jusZsb0YQK2A==" + }, "builtin-modules": { "version": "3.1.0", "resolved": "https://registry.npmjs.org/builtin-modules/-/builtin-modules-3.1.0.tgz", @@ -1210,6 +1371,11 @@ "resolved": "https://registry.npmjs.org/charenc/-/charenc-0.0.2.tgz", "integrity": "sha1-wKHS86cJLgN3S/qD8UwPxXkKhmc=" }, + "chownr": { + "version": "1.1.4", + "resolved": "https://registry.npmjs.org/chownr/-/chownr-1.1.4.tgz", + "integrity": "sha512-jJ0bqzaylmJtVnNgzTeSOs8DPavpbYgEr/b0YL8/2GO3xJEhInFmhKMUnEJQjZumK7KXGFhUy89PrsJWlakBVg==" + }, "cli-cursor": { "version": "3.1.0", "resolved": "https://registry.npmjs.org/cli-cursor/-/cli-cursor-3.1.0.tgz", @@ -1266,10 +1432,10 @@ "resolved": "https://registry.npmjs.org/cluster-key-slot/-/cluster-key-slot-1.1.0.tgz", "integrity": "sha512-2Nii8p3RwAPiFwsnZvukotvow2rIHM+yQ6ZcBXGHdniadkYGZYiGmkHJIbZPIV9nfv7m/U1IPMVVcAhoWFeklw==" }, - "coffee-script": { - "version": "1.6.0", - "resolved": "https://registry.npmjs.org/coffee-script/-/coffee-script-1.6.0.tgz", - "integrity": "sha512-Tx8itEfCsQp8RbLDFt7qwjqXycAx2g6SI7//4PPUR2j6meLmNifYm6zKrNDcU1+Q/GWRhjhEZk7DaLG1TfIzGA==" + "code-point-at": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/code-point-at/-/code-point-at-1.1.0.tgz", + "integrity": "sha1-DQcLTQQ6W+ozovGkDi7bPZpMz3c=" }, "color-convert": { "version": "1.9.3", @@ -1305,6 +1471,47 @@ "integrity": "sha512-6P6g0uetGpW/sdyUy/iQQCbFF0kWVMSIVSyYz7Zgjcgh8mgw8PQzDNZeyZ5DQ2gM7LBoZPHmnjz8rUthkBG5tw==", "dev": true }, + "compressible": { + "version": "2.0.18", + "resolved": "https://registry.npmjs.org/compressible/-/compressible-2.0.18.tgz", + "integrity": "sha512-AF3r7P5dWxL8MxyITRMlORQNaOA2IkAFaTr4k7BUumjPtRpGDTZpl0Pb1XCO6JeDCBdp126Cgs9sMxqSjgYyRg==", + "requires": { + "mime-db": ">= 1.43.0 < 2" + }, + "dependencies": { + "mime-db": { + "version": "1.45.0", + "resolved": "https://registry.npmjs.org/mime-db/-/mime-db-1.45.0.tgz", + "integrity": "sha512-CkqLUxUk15hofLoLyljJSrukZi8mAtgd+yE5uO4tqRZsdsAJKv0O+rFMhVDRJgozy+yG6md5KwuXhD4ocIoP+w==" + } + } + }, + "compression": { + "version": "1.7.4", + "resolved": "https://registry.npmjs.org/compression/-/compression-1.7.4.tgz", + "integrity": "sha512-jaSIDzP9pZVS4ZfQ+TzvtiWhdpFhE2RDHz8QJkpX9SIpLq88VueF5jJw6t+6CUQcAoA6t+x89MLrWAqpfDE8iQ==", + "requires": { + "accepts": "~1.3.5", + "bytes": "3.0.0", + "compressible": "~2.0.16", + "debug": "2.6.9", + "on-headers": "~1.0.2", + "safe-buffer": "5.1.2", + "vary": "~1.1.2" + }, + "dependencies": { + "bytes": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/bytes/-/bytes-3.0.0.tgz", + "integrity": "sha1-0ygVQE1olpn4Wk6k+odV3ROpYEg=" + }, + "safe-buffer": { + "version": "5.1.2", + "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.1.2.tgz", + "integrity": "sha512-Gd2UZBJDkXlY7GbJxfsE8/nvKkUEU1G38c1siN6QP6a9PT9MmHB8GnpscSmMJSoF8LOIrt8ud/wPtojys4G6+g==" + } + } + }, "concat-map": { "version": "0.0.1", "resolved": "https://registry.npmjs.org/concat-map/-/concat-map-0.0.1.tgz", @@ -1329,6 +1536,11 @@ } } }, + "console-control-strings": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/console-control-strings/-/console-control-strings-1.1.0.tgz", + "integrity": "sha1-PXz0Rk22RG6mRL9LOVB/mFEAjo4=" + }, "console-log-level": { "version": "1.4.1", "resolved": "https://registry.npmjs.org/console-log-level/-/console-log-level-1.4.1.tgz", @@ -1474,6 +1686,11 @@ "type-detect": "0.1.1" } }, + "deep-extend": { + "version": "0.6.0", + "resolved": "https://registry.npmjs.org/deep-extend/-/deep-extend-0.6.0.tgz", + "integrity": "sha512-LOHxIOaPYdHlJRtCQfDIVZtfw/ufM8+rVj649RIHzcm/vGwQRXFt6OPqIFWsm2XEMrNIEtWR64sY1LEKD2vAOA==" + }, "deep-is": { "version": "0.1.3", "resolved": "https://registry.npmjs.org/deep-is/-/deep-is-0.1.3.tgz", @@ -1499,6 +1716,11 @@ "resolved": "https://registry.npmjs.org/delayed-stream/-/delayed-stream-1.0.0.tgz", "integrity": "sha512-ZySD7Nf91aLB0RxL4KGrKHBXl7Eds1DAmEdcoVawXnLD7SDhpNgtuII2aAkg7a7QS41jxPSZ17p4VdGnMHk3MQ==" }, + "delegates": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/delegates/-/delegates-1.0.0.tgz", + "integrity": "sha1-hMbhWbgZBP3KWaDvRM2HDTElD5o=" + }, "denque": { "version": "1.4.1", "resolved": "https://registry.npmjs.org/denque/-/denque-1.4.1.tgz", @@ -1514,6 +1736,11 @@ "resolved": "https://registry.npmjs.org/destroy/-/destroy-1.0.4.tgz", "integrity": "sha512-3NdhDuEXnfun/z7x9GOElY49LoqVHoGScmOKwmxhsS8N5Y+Z8KyPPDnaSzqWgYt/ji4mqwfTS34Htrk0zPIXVg==" }, + "detect-libc": { + "version": "1.0.3", + "resolved": "https://registry.npmjs.org/detect-libc/-/detect-libc-1.0.3.tgz", + "integrity": "sha1-+hN8S9aY7fVc1c0CrFWfkaTEups=" + }, "diff": { "version": "3.5.0", "resolved": "https://registry.npmjs.org/diff/-/diff-3.5.0.tgz", @@ -1654,19 +1881,6 @@ "is-symbol": "^1.0.2" } }, - "es6-promise": { - "version": "4.2.8", - "resolved": "https://registry.npmjs.org/es6-promise/-/es6-promise-4.2.8.tgz", - "integrity": "sha512-HJDGx5daxeIvxdBxvG2cb9g4tEvwIk3i8+nhX0yGrYmZUzbkdg8QbDevheDB8gd0//uPj4c1EQua8Q+MViT0/w==" - }, - "es6-promisify": { - "version": "5.0.0", - "resolved": "https://registry.npmjs.org/es6-promisify/-/es6-promisify-5.0.0.tgz", - "integrity": "sha512-C+d6UdsYDk0lMebHNR4S2NybQMMngAOnOwYBQjTOiv0MkoJMP0Myw2mgpDLBcpfCmRLxyFqYhS/CfOENq4SJhQ==", - "requires": { - "es6-promise": "^4.0.3" - } - }, "escape-html": { "version": "1.0.3", "resolved": "https://registry.npmjs.org/escape-html/-/escape-html-1.0.3.tgz", @@ -2275,24 +2489,6 @@ "integrity": "sha512-r5wGx7YeOwNWNlCA0wQ86zKyDLMQr+/RB8xy74M4hTphfmjlijTSSXGuH8rnvKZnfT9i+75zmd8jcKdMR4O6jA==", "dev": true }, - "follow-redirects": { - "version": "1.5.10", - "resolved": "https://registry.npmjs.org/follow-redirects/-/follow-redirects-1.5.10.tgz", - "integrity": "sha512-0V5l4Cizzvqt5D44aTXbFZz+FtyXV1vrDN6qrelxtfYQKW0KO0W2T/hkE8xvGa/540LkZlkaUjO4ailYTFtHVQ==", - "requires": { - "debug": "=3.1.0" - }, - "dependencies": { - "debug": { - "version": "3.1.0", - "resolved": "https://registry.npmjs.org/debug/-/debug-3.1.0.tgz", - "integrity": "sha512-OX8XqP7/1a9cqkxYw2yXss15f26NKWBpDXQd0/uK/KPqdQhxbPa994hnzjcE2VqQpDslf55723cKPUOGSmMY3g==", - "requires": { - "ms": "2.0.0" - } - } - } - }, "forever-agent": { "version": "0.6.1", "resolved": "https://registry.npmjs.org/forever-agent/-/forever-agent-0.6.1.tgz", @@ -2327,11 +2523,18 @@ "resolved": "https://registry.npmjs.org/fresh/-/fresh-0.5.2.tgz", "integrity": "sha512-zJ2mQYM18rEFOudeV4GShTGIQ7RbzA7ozbU9I/XBpm7kqgMywgmylMwXHxZJmkVoYkna9d2pVXVXPdYTP9ej8Q==" }, + "fs-minipass": { + "version": "1.2.7", + "resolved": "https://registry.npmjs.org/fs-minipass/-/fs-minipass-1.2.7.tgz", + "integrity": "sha512-GWSSJGFy4e9GUeCcbIkED+bgAoFyj7XF1mV8rma3QW4NIqX9Kyx79N/PF61H5udOV3aY1IaMLs6pGbH71nlCTA==", + "requires": { + "minipass": "^2.6.0" + } + }, "fs.realpath": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/fs.realpath/-/fs.realpath-1.0.0.tgz", - "integrity": "sha512-OO0pH2lK6a0hZnAdau5ItzHPI6pUlvI7jMVnxUQRtw4owF2wk8lOSabtGDCTP4Ggrg2MbGnWO9X8K1t4+fGMDw==", - "dev": true + "integrity": "sha512-OO0pH2lK6a0hZnAdau5ItzHPI6pUlvI7jMVnxUQRtw4owF2wk8lOSabtGDCTP4Ggrg2MbGnWO9X8K1t4+fGMDw==" }, "function-bind": { "version": "1.1.1", @@ -2345,24 +2548,52 @@ "integrity": "sha512-dsKNQNdj6xA3T+QlADDA7mOSlX0qiMINjn0cgr+eGHGsbSHzTabcIogz2+p/iqP1Xs6EP/sS2SbqH+brGTbq0g==", "dev": true }, - "gaxios": { - "version": "1.8.4", - "resolved": "https://registry.npmjs.org/gaxios/-/gaxios-1.8.4.tgz", - "integrity": "sha512-BoENMnu1Gav18HcpV9IleMPZ9exM+AvUjrAOV4Mzs/vfz2Lu/ABv451iEXByKiMPn2M140uul1txXCg83sAENw==", + "gauge": { + "version": "2.7.4", + "resolved": "https://registry.npmjs.org/gauge/-/gauge-2.7.4.tgz", + "integrity": "sha1-LANAXHU4w51+s3sxcCLjJfsBi/c=", "requires": { - "abort-controller": "^3.0.0", - "extend": "^3.0.2", - "https-proxy-agent": "^2.2.1", - "node-fetch": "^2.3.0" - } - }, - "gcp-metadata": { - "version": "1.0.0", - "resolved": "https://registry.npmjs.org/gcp-metadata/-/gcp-metadata-1.0.0.tgz", - "integrity": "sha512-Q6HrgfrCQeEircnNP3rCcEgiDv7eF9+1B+1MMgpE190+/+0mjQR8PxeOaRgxZWmdDAF9EIryHB9g1moPiw1SbQ==", - "requires": { - "gaxios": "^1.0.2", - "json-bigint": "^0.3.0" + "aproba": "^1.0.3", + "console-control-strings": "^1.0.0", + "has-unicode": "^2.0.0", + "object-assign": "^4.1.0", + "signal-exit": "^3.0.0", + "string-width": "^1.0.1", + "strip-ansi": "^3.0.1", + "wide-align": "^1.1.0" + }, + "dependencies": { + "ansi-regex": { + "version": "2.1.1", + "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-2.1.1.tgz", + "integrity": "sha1-w7M6te42DYbg5ijwRorn7yfWVN8=" + }, + "is-fullwidth-code-point": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/is-fullwidth-code-point/-/is-fullwidth-code-point-1.0.0.tgz", + "integrity": "sha1-754xOG8DGn8NZDr4L95QxFfvAMs=", + "requires": { + "number-is-nan": "^1.0.0" + } + }, + "string-width": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/string-width/-/string-width-1.0.2.tgz", + "integrity": "sha1-EYvfW4zcUaKn5w0hHgfisLmxB9M=", + "requires": { + "code-point-at": "^1.0.0", + "is-fullwidth-code-point": "^1.0.0", + "strip-ansi": "^3.0.0" + } + }, + "strip-ansi": { + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-3.0.1.tgz", + "integrity": "sha1-ajhfuIU9lS1f8F0Oiq+UJ43GPc8=", + "requires": { + "ansi-regex": "^2.0.0" + } + } } }, "get-caller-file": { @@ -2416,29 +2647,6 @@ "type-fest": "^0.8.1" } }, - "google-auth-library": { - "version": "3.1.2", - "resolved": "https://registry.npmjs.org/google-auth-library/-/google-auth-library-3.1.2.tgz", - "integrity": "sha512-cDQMzTotwyWMrg5jRO7q0A4TL/3GWBgO7I7q5xGKNiiFf9SmGY/OJ1YsLMgI2MVHHsEGyrqYnbnmV1AE+Z6DnQ==", - "requires": { - "base64-js": "^1.3.0", - "fast-text-encoding": "^1.0.0", - "gaxios": "^1.2.1", - "gcp-metadata": "^1.0.0", - "gtoken": "^2.3.2", - "https-proxy-agent": "^2.2.1", - "jws": "^3.1.5", - "lru-cache": "^5.0.0", - "semver": "^5.5.0" - }, - "dependencies": { - "semver": { - "version": "5.7.1", - "resolved": "https://registry.npmjs.org/semver/-/semver-5.7.1.tgz", - "integrity": "sha512-sauaDf/PZdVgrLTNYHRtpXa1iRiKcaebiKQ1BJdpQlWH2lCvexQdX55snPFyK7QzpudqbCI0qXFfOasHdyNDGQ==" - } - } - }, "google-gax": { "version": "1.15.3", "resolved": "https://registry.npmjs.org/google-gax/-/google-gax-1.15.3.tgz", @@ -2578,40 +2786,12 @@ } } }, - "google-p12-pem": { - "version": "1.0.4", - "resolved": "https://registry.npmjs.org/google-p12-pem/-/google-p12-pem-1.0.4.tgz", - "integrity": "sha512-SwLAUJqUfTB2iS+wFfSS/G9p7bt4eWcc2LyfvmUXe7cWp6p3mpxDo6LLI29MXdU6wvPcQ/up298X7GMC5ylAlA==", - "requires": { - "node-forge": "^0.8.0", - "pify": "^4.0.0" - } - }, "graceful-fs": { "version": "4.2.4", "resolved": "https://registry.npmjs.org/graceful-fs/-/graceful-fs-4.2.4.tgz", "integrity": "sha512-WjKPNJF79dtJAVniUlGGWHYGz2jWxT6VhN/4m1NdkbZ2nOsEF+cI1Edgql5zCRhs/VsQYRvrXctxktVXZUkixw==", "dev": true }, - "gtoken": { - "version": "2.3.3", - "resolved": "https://registry.npmjs.org/gtoken/-/gtoken-2.3.3.tgz", - "integrity": "sha512-EaB49bu/TCoNeQjhCYKI/CurooBKkGxIqFHsWABW0b25fobBYVTMe84A8EBVVZhl8emiUdNypil9huMOTmyAnw==", - "requires": { - "gaxios": "^1.0.4", - "google-p12-pem": "^1.0.0", - "jws": "^3.1.5", - "mime": "^2.2.0", - "pify": "^4.0.0" - }, - "dependencies": { - "mime": { - "version": "2.4.4", - "resolved": "https://registry.npmjs.org/mime/-/mime-2.4.4.tgz", - "integrity": "sha512-LRxmNwziLPT828z+4YkNzloCFC2YM4wrB99k+AV5ZbEyfGNWfG8SO1FUXLmLDBSo89NrJZ4DIWeLjy1CHGhMGA==" - } - } - }, "har-schema": { "version": "2.0.0", "resolved": "https://registry.npmjs.org/har-schema/-/har-schema-2.0.0.tgz", @@ -2664,6 +2844,11 @@ "integrity": "sha512-PLcsoqu++dmEIZB+6totNFKq/7Do+Z0u4oT0zKOJNl3lYK6vGwwu2hjHs+68OEZbTjiUE9bgOABXbP/GvrS0Kg==", "dev": true }, + "has-unicode": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/has-unicode/-/has-unicode-2.0.1.tgz", + "integrity": "sha1-4Ob+aijPUROIVeCG0Wkedx3iqLk=" + }, "he": { "version": "1.1.1", "resolved": "https://registry.npmjs.org/he/-/he-1.1.1.tgz", @@ -2736,30 +2921,6 @@ "sshpk": "^1.7.0" } }, - "https-proxy-agent": { - "version": "2.2.4", - "resolved": "https://registry.npmjs.org/https-proxy-agent/-/https-proxy-agent-2.2.4.tgz", - "integrity": "sha512-OmvfoQ53WLjtA9HeYP9RNrWMJzzAz1JGaSFr1nijg0PVR1JaD/xbJq1mdEIIlxGpXp9eSe/O2LgU9DJmTPd0Eg==", - "requires": { - "agent-base": "^4.3.0", - "debug": "^3.1.0" - }, - "dependencies": { - "debug": { - "version": "3.2.6", - "resolved": "https://registry.npmjs.org/debug/-/debug-3.2.6.tgz", - "integrity": "sha512-mel+jf7nrtEl5Pn1Qx46zARXKDpBbvzezse7p7LqINmdoIk8PYP5SySaxEmYv6TZ0JyEKA1hsCId6DIhgITtWQ==", - "requires": { - "ms": "^2.1.1" - } - }, - "ms": { - "version": "2.1.2", - "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.2.tgz", - "integrity": "sha512-sGkPx+VjMtmA6MX27oA4FBFELFCZZ4S4XqeGOXCv68tT+jb3vk/RyaKWP0PTKyWtmLSM0b+adUTEvbs1PEaH2w==" - } - } - }, "iconv-lite": { "version": "0.4.24", "resolved": "https://registry.npmjs.org/iconv-lite/-/iconv-lite-0.4.24.tgz", @@ -2774,6 +2935,14 @@ "integrity": "sha512-cyFDKrqc/YdcWFniJhzI42+AzS+gNwmUzOSFcRCQYwySuBBBy/KjuxWLZ/FHEH6Moq1NizMOBWyTcv8O4OZIMg==", "dev": true }, + "ignore-walk": { + "version": "3.0.3", + "resolved": "https://registry.npmjs.org/ignore-walk/-/ignore-walk-3.0.3.tgz", + "integrity": "sha512-m7o6xuOaT1aqheYHKf8W6J5pYH85ZI9w077erOzLje3JsB1gkafkAhHHY19dqjulgIZHFm32Cp5uNZgcQqdJKw==", + "requires": { + "minimatch": "^3.0.4" + } + }, "import-fresh": { "version": "3.2.1", "resolved": "https://registry.npmjs.org/import-fresh/-/import-fresh-3.2.1.tgz", @@ -2810,6 +2979,11 @@ "resolved": "https://registry.npmjs.org/inherits/-/inherits-2.0.3.tgz", "integrity": "sha512-x00IRNXNy63jwGkJmzPigoySHbaqpNuzKbBOmzK+g2OdZpQ9w+sxCN+VSB3ja7IAge2OP2qpfxTjeNcyjmW1uw==" }, + "ini": { + "version": "1.3.5", + "resolved": "https://registry.npmjs.org/ini/-/ini-1.3.5.tgz", + "integrity": "sha512-RZY5huIKCMRWDUqZlEi72f/lmXKMvuszcMBduliQ3nnWbx9X/ZBQO7DijMEYS9EhHBb2qacRUMtC7svLwe0lcw==" + }, "inquirer": { "version": "7.2.0", "resolved": "https://registry.npmjs.org/inquirer/-/inquirer-7.2.0.tgz", @@ -2939,11 +3113,6 @@ "integrity": "sha512-zz06S8t0ozoDXMG+ube26zeCTNXcKIPJZJi8hBrF4idCLms4CG9QtK7qBl1boi5ODzFpjswb5JPmHCbMpjaYzg==", "dev": true }, - "is-buffer": { - "version": "2.0.4", - "resolved": "https://registry.npmjs.org/is-buffer/-/is-buffer-2.0.4.tgz", - "integrity": "sha512-Kq1rokWXOPXWuaMAqZiJW4XxsmD9zGx9q4aePabbn3qCRGedtH7Cm+zV8WETitMfu1wdh+Rvd6w5egwSngUX2A==" - }, "is-callable": { "version": "1.2.0", "resolved": "https://registry.npmjs.org/is-callable/-/is-callable-1.2.0.tgz", @@ -3115,25 +3284,6 @@ "verror": "1.10.0" } }, - "jwa": { - "version": "1.4.1", - "resolved": "https://registry.npmjs.org/jwa/-/jwa-1.4.1.tgz", - "integrity": "sha512-qiLX/xhEEFKUAJ6FiBMbes3w9ATzyk5W7Hvzpa/SLYdxNtng+gcurvrI7TbACjIXlsJyr05/S1oUhZrc63evQA==", - "requires": { - "buffer-equal-constant-time": "1.0.1", - "ecdsa-sig-formatter": "1.0.11", - "safe-buffer": "^5.0.1" - } - }, - "jws": { - "version": "3.2.2", - "resolved": "https://registry.npmjs.org/jws/-/jws-3.2.2.tgz", - "integrity": "sha512-YHlZCB6lMTllWDtSPHz/ZXTsi8S00usEV6v1tjq8tOUZzw7DpSDWVXjXDre6ed1w/pd495ODpHZYSdkRTsa0HA==", - "requires": { - "jwa": "^1.4.1", - "safe-buffer": "^5.0.1" - } - }, "levn": { "version": "0.3.0", "resolved": "https://registry.npmjs.org/levn/-/levn-0.3.0.tgz", @@ -3217,11 +3367,6 @@ "integrity": "sha512-0KpjqXRVvrYyCsX1swR/XTK0va6VQkQM6MNo7PqW77ByjAhoARA8EfrP1N4+KlKj8YS0ZUCtRT/YUuhyYDujIQ==", "dev": true }, - "lodash.pickby": { - "version": "4.6.0", - "resolved": "https://registry.npmjs.org/lodash.pickby/-/lodash.pickby-4.6.0.tgz", - "integrity": "sha512-AZV+GsS/6ckvPOVQPXSiFFacKvKB4kOQu6ynt9wz0F3LO4R9Ij4K1ddYsIytDpSgLz88JHd9P+oaLeej5/Sl7Q==" - }, "lodash.unescape": { "version": "4.0.1", "resolved": "https://registry.npmjs.org/lodash.unescape/-/lodash.unescape-4.0.1.tgz", @@ -3356,15 +3501,6 @@ "yallist": "^3.0.2" } }, - "lynx": { - "version": "0.1.1", - "resolved": "https://registry.npmjs.org/lynx/-/lynx-0.1.1.tgz", - "integrity": "sha512-JI52N0NwK2b/Md0TFPdPtUBI46kjyJXF7+q08l2yvQ56q6QA8s7ZjZQQRoxFpS2jDXNf/B0p8ID+OIKcTsZwzw==", - "requires": { - "mersenne": "~0.0.3", - "statsd-parser": "~0.0.4" - } - }, "make-plural": { "version": "4.3.0", "resolved": "https://registry.npmjs.org/make-plural/-/make-plural-4.3.0.tgz", @@ -3415,11 +3551,6 @@ "resolved": "https://registry.npmjs.org/merge-descriptors/-/merge-descriptors-1.0.1.tgz", "integrity": "sha512-cCi6g3/Zr1iqQi6ySbseM1Xvooa98N0w31jzUYrXPX2xqObmFGHJ0tQ5u74H3mVh7wLouTseZyYIq39g8cNp1w==" }, - "mersenne": { - "version": "0.0.4", - "resolved": "https://registry.npmjs.org/mersenne/-/mersenne-0.0.4.tgz", - "integrity": "sha512-XoSUL+nF8hMTKGQxUs8r3Btdsf1yuKKBdCCGbh3YXgCXuVKishpZv1CNc385w9s8t4Ynwc5h61BwW/FCVulkbg==" - }, "messageformat": { "version": "2.3.0", "resolved": "https://registry.npmjs.org/messageformat/-/messageformat-2.3.0.tgz", @@ -3448,28 +3579,6 @@ "resolved": "https://registry.npmjs.org/methods/-/methods-1.1.2.tgz", "integrity": "sha512-iclAHeNqNm68zFtnZ0e+1L2yUIdvzNoauKU4WBA3VvH/vPFieF7qfRlwUZU+DA9P9bPXIS90ulxoUoCH23sV2w==" }, - "metrics-sharelatex": { - "version": "2.6.2", - "resolved": "https://registry.npmjs.org/metrics-sharelatex/-/metrics-sharelatex-2.6.2.tgz", - "integrity": "sha512-bOLfkSCexiPgB96hdXhoOWyvvrwscgjeZPEqdcJ7BTGxY59anzvymNf5hTGJ1RtS4sblDKxITw3L5a+gYKhRYQ==", - "requires": { - "@google-cloud/debug-agent": "^3.0.0", - "@google-cloud/profiler": "^0.2.3", - "@google-cloud/trace-agent": "^3.2.0", - "coffee-script": "1.6.0", - "lynx": "~0.1.1", - "prom-client": "^11.1.3", - "underscore": "~1.6.0", - "yn": "^3.1.1" - }, - "dependencies": { - "underscore": { - "version": "1.6.0", - "resolved": "https://registry.npmjs.org/underscore/-/underscore-1.6.0.tgz", - "integrity": "sha512-z4o1fvKUojIWh9XuaVLUDdf86RQiq13AC1dmHbTpoyuu+bquHms76v16CjycCbec87J7z0k//SiQVk0sMdFmpQ==" - } - } - }, "mime": { "version": "1.6.0", "resolved": "https://registry.npmjs.org/mime/-/mime-1.6.0.tgz", @@ -3507,6 +3616,30 @@ "resolved": "https://registry.npmjs.org/minimist/-/minimist-0.0.8.tgz", "integrity": "sha512-miQKw5Hv4NS1Psg2517mV4e4dYNaO3++hjAvLOAzKqZ61rH8NS1SK+vbfBWZ5PY/Me/bEWhUwqMghEW5Fb9T7Q==" }, + "minipass": { + "version": "2.9.0", + "resolved": "https://registry.npmjs.org/minipass/-/minipass-2.9.0.tgz", + "integrity": "sha512-wxfUjg9WebH+CUDX/CdbRlh5SmfZiy/hpkxaRI16Y9W56Pa75sWgd/rvFilSgrauD9NyFymP/+JFV3KwzIsJeg==", + "requires": { + "safe-buffer": "^5.1.2", + "yallist": "^3.0.0" + }, + "dependencies": { + "safe-buffer": { + "version": "5.2.1", + "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.2.1.tgz", + "integrity": "sha512-rp3So07KcdmmKbGvgaNxQSJr7bGVSVk5S9Eq1F+ppbRo70+YeaDxkw5Dd8NPN+GD6bjnYm2VuPuCXmpuYvmCXQ==" + } + } + }, + "minizlib": { + "version": "1.3.3", + "resolved": "https://registry.npmjs.org/minizlib/-/minizlib-1.3.3.tgz", + "integrity": "sha512-6ZYMOEnmVsdCeTJVE0W9ZD+pVnE8h9Hma/iOwwRDsdQoePpoX56/8B6z3P9VNwppJuBKNRuFDRNRqRWexT9G9Q==", + "requires": { + "minipass": "^2.9.0" + } + }, "mkdirp": { "version": "0.5.1", "resolved": "https://registry.npmjs.org/mkdirp/-/mkdirp-0.5.1.tgz", @@ -3624,11 +3757,6 @@ "rimraf": "~2.4.0" } }, - "nan": { - "version": "2.12.1", - "resolved": "https://registry.npmjs.org/nan/-/nan-2.12.1.tgz", - "integrity": "sha512-JY7V6lRkStKcKTvHO5NVSQRv+RV+FIL5pvDoLiAtSL9pKlC5x9PKQcZDsq7m4FO4d57mkhC6Z+QhAh3Jdk5JFw==" - }, "native-promise-only": { "version": "0.8.1", "resolved": "https://registry.npmjs.org/native-promise-only/-/native-promise-only-0.8.1.tgz", @@ -3647,6 +3775,31 @@ "integrity": "sha512-zIdGUrPRFTUELUvr3Gmc7KZ2Sw/h1PiVM0Af/oHB6zgnV1ikqSfRk+TOufi79aHYCW3NiOXmr1BP5nWbzojLaA==", "optional": true }, + "needle": { + "version": "2.5.2", + "resolved": "https://registry.npmjs.org/needle/-/needle-2.5.2.tgz", + "integrity": "sha512-LbRIwS9BfkPvNwNHlsA41Q29kL2L/6VaOJ0qisM5lLWsTV3nP15abO5ITL6L81zqFhzjRKDAYjpcBcwM0AVvLQ==", + "requires": { + "debug": "^3.2.6", + "iconv-lite": "^0.4.4", + "sax": "^1.2.4" + }, + "dependencies": { + "debug": { + "version": "3.2.7", + "resolved": "https://registry.npmjs.org/debug/-/debug-3.2.7.tgz", + "integrity": "sha512-CFjzYYAi4ThfiQvizrFQevTTXHtnCqWfe7x1AhgEscTz6ZbLbfoLRLPugTQyBth6f8ZERVUSyWHFD/7Wu4t1XQ==", + "requires": { + "ms": "^2.1.1" + } + }, + "ms": { + "version": "2.1.2", + "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.2.tgz", + "integrity": "sha512-sGkPx+VjMtmA6MX27oA4FBFELFCZZ4S4XqeGOXCv68tT+jb3vk/RyaKWP0PTKyWtmLSM0b+adUTEvbs1PEaH2w==" + } + } + }, "negotiator": { "version": "0.6.2", "resolved": "https://registry.npmjs.org/negotiator/-/negotiator-0.6.2.tgz", @@ -3663,10 +3816,72 @@ "resolved": "https://registry.npmjs.org/node-fetch/-/node-fetch-2.6.1.tgz", "integrity": "sha512-V4aYg89jEoVRxRb2fJdAg8FHvI7cEyYdVAh94HH0UIK8oJxUfkjlDQN9RbMx+bEjP7+ggMiFRprSti032Oipxw==" }, - "node-forge": { - "version": "0.8.5", - "resolved": "https://registry.npmjs.org/node-forge/-/node-forge-0.8.5.tgz", - "integrity": "sha512-vFMQIWt+J/7FLNyKouZ9TazT74PRV3wgv9UT4cRjC8BffxFbKXkgIWR42URCPSnHm/QDz6BOlb2Q0U4+VQT67Q==" + "node-pre-gyp": { + "version": "0.16.0", + "resolved": "https://registry.npmjs.org/node-pre-gyp/-/node-pre-gyp-0.16.0.tgz", + "integrity": "sha512-4efGA+X/YXAHLi1hN8KaPrILULaUn2nWecFrn1k2I+99HpoyvcOGEbtcOxpDiUwPF2ZANMJDh32qwOUPenuR1g==", + "requires": { + "detect-libc": "^1.0.2", + "mkdirp": "^0.5.3", + "needle": "^2.5.0", + "nopt": "^4.0.1", + "npm-packlist": "^1.1.6", + "npmlog": "^4.0.2", + "rc": "^1.2.7", + "rimraf": "^2.6.1", + "semver": "^5.3.0", + "tar": "^4.4.2" + }, + "dependencies": { + "glob": { + "version": "7.1.6", + "resolved": "https://registry.npmjs.org/glob/-/glob-7.1.6.tgz", + "integrity": "sha512-LwaxwyZ72Lk7vZINtNNrywX0ZuLyStrdDtabefZKAY5ZGJhVtgdznluResxNmPitE0SAO+O26sWTHeKSI2wMBA==", + "requires": { + "fs.realpath": "^1.0.0", + "inflight": "^1.0.4", + "inherits": "2", + "minimatch": "^3.0.4", + "once": "^1.3.0", + "path-is-absolute": "^1.0.0" + } + }, + "minimist": { + "version": "1.2.5", + "resolved": "https://registry.npmjs.org/minimist/-/minimist-1.2.5.tgz", + "integrity": "sha512-FM9nNUYrRBAELZQT3xeZQ7fmMOBg6nWNmJKTcgsJeaLstP/UODVpGsr5OhXhhXg6f+qtJ8uiZ+PUxkDWcgIXLw==" + }, + "mkdirp": { + "version": "0.5.5", + "resolved": "https://registry.npmjs.org/mkdirp/-/mkdirp-0.5.5.tgz", + "integrity": "sha512-NKmAlESf6jMGym1++R0Ra7wvhV+wFW63FaSOFPwRahvea0gMUcGUhVeAg/0BC0wiv9ih5NYPB1Wn1UEI1/L+xQ==", + "requires": { + "minimist": "^1.2.5" + } + }, + "rimraf": { + "version": "2.7.1", + "resolved": "https://registry.npmjs.org/rimraf/-/rimraf-2.7.1.tgz", + "integrity": "sha512-uWjbaKIK3T1OSVptzX7Nl6PvQ3qAGtKEtVRjRuazjfL3Bx5eI409VZSqgND+4UNnmzLVdPj9FqFJNPqBZFve4w==", + "requires": { + "glob": "^7.1.3" + } + }, + "semver": { + "version": "5.7.1", + "resolved": "https://registry.npmjs.org/semver/-/semver-5.7.1.tgz", + "integrity": "sha512-sauaDf/PZdVgrLTNYHRtpXa1iRiKcaebiKQ1BJdpQlWH2lCvexQdX55snPFyK7QzpudqbCI0qXFfOasHdyNDGQ==" + } + } + }, + "nopt": { + "version": "4.0.3", + "resolved": "https://registry.npmjs.org/nopt/-/nopt-4.0.3.tgz", + "integrity": "sha512-CvaGwVMztSMJLOeXPrez7fyfObdZqNUK1cPAEzLHrTybIua9pMdmmPR5YwtfNftIOMv3DPUhFaxsZMNTQO20Kg==", + "requires": { + "abbrev": "1", + "osenv": "^0.1.4" + } }, "normalize-package-data": { "version": "2.5.0", @@ -3688,11 +3903,55 @@ } } }, + "npm-bundled": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/npm-bundled/-/npm-bundled-1.1.1.tgz", + "integrity": "sha512-gqkfgGePhTpAEgUsGEgcq1rqPXA+tv/aVBlgEzfXwA1yiUJF7xtEt3CtVwOjNYQOVknDk0F20w58Fnm3EtG0fA==", + "requires": { + "npm-normalize-package-bin": "^1.0.1" + } + }, + "npm-normalize-package-bin": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/npm-normalize-package-bin/-/npm-normalize-package-bin-1.0.1.tgz", + "integrity": "sha512-EPfafl6JL5/rU+ot6P3gRSCpPDW5VmIzX959Ob1+ySFUuuYHWHekXpwdUZcKP5C+DS4GEtdJluwBjnsNDl+fSA==" + }, + "npm-packlist": { + "version": "1.4.8", + "resolved": "https://registry.npmjs.org/npm-packlist/-/npm-packlist-1.4.8.tgz", + "integrity": "sha512-5+AZgwru5IevF5ZdnFglB5wNlHG1AOOuw28WhUq8/8emhBmLv6jX5by4WJCh7lW0uSYZYS6DXqIsyZVIXRZU9A==", + "requires": { + "ignore-walk": "^3.0.1", + "npm-bundled": "^1.0.1", + "npm-normalize-package-bin": "^1.0.1" + } + }, + "npmlog": { + "version": "4.1.2", + "resolved": "https://registry.npmjs.org/npmlog/-/npmlog-4.1.2.tgz", + "integrity": "sha512-2uUqazuKlTaSI/dC8AzicUck7+IrEaOnN/e0jd3Xtt1KcGpwx30v50mL7oPyr/h9bL3E4aZccVwpwP+5W9Vjkg==", + "requires": { + "are-we-there-yet": "~1.1.2", + "console-control-strings": "~1.1.0", + "gauge": "~2.7.3", + "set-blocking": "~2.0.0" + } + }, + "number-is-nan": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/number-is-nan/-/number-is-nan-1.0.1.tgz", + "integrity": "sha1-CXtgK1NCKlIsGvuHkDGDNpQaAR0=" + }, "oauth-sign": { "version": "0.9.0", "resolved": "https://registry.npmjs.org/oauth-sign/-/oauth-sign-0.9.0.tgz", "integrity": "sha512-fexhUFFPTGV8ybAtSIGbV6gOkSv8UtRbDBnAyLQw4QPKkgNlsH2ByPGtMUqdWkos6YCRmAqViwgZrJc/mRDzZQ==" }, + "object-assign": { + "version": "4.1.1", + "resolved": "https://registry.npmjs.org/object-assign/-/object-assign-4.1.1.tgz", + "integrity": "sha1-IQmtx5ZYh8/AXLvUQsrIv7s2CGM=" + }, "object-inspect": { "version": "1.8.0", "resolved": "https://registry.npmjs.org/object-inspect/-/object-inspect-1.8.0.tgz", @@ -3778,16 +4037,30 @@ "resolved": "https://registry.npmjs.org/options/-/options-0.0.6.tgz", "integrity": "sha1-7CLTEoBrtT5zF3Pnza788cZDEo8=" }, + "os-homedir": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/os-homedir/-/os-homedir-1.0.2.tgz", + "integrity": "sha1-/7xJiDNuDoM94MFox+8VISGqf7M=" + }, "os-tmpdir": { "version": "1.0.2", "resolved": "https://registry.npmjs.org/os-tmpdir/-/os-tmpdir-1.0.2.tgz", - "integrity": "sha512-D2FR03Vir7FIu45XBY20mTb+/ZSWB00sjU9jdQXt83gDrI4Ztz5Fs7/yy74g2N5SVQY4xY1qDr4rNddwYRVX0g==", - "dev": true + "integrity": "sha512-D2FR03Vir7FIu45XBY20mTb+/ZSWB00sjU9jdQXt83gDrI4Ztz5Fs7/yy74g2N5SVQY4xY1qDr4rNddwYRVX0g==" + }, + "osenv": { + "version": "0.1.5", + "resolved": "https://registry.npmjs.org/osenv/-/osenv-0.1.5.tgz", + "integrity": "sha512-0CWcCECdMVc2Rw3U5w9ZjqX6ga6ubk1xDVKxtBQPK7wis/0F2r9T6k4ydGYhecl7YUBxBVxhL5oisPsNxAPe2g==", + "requires": { + "os-homedir": "^1.0.0", + "os-tmpdir": "^1.0.0" + } }, "p-limit": { "version": "2.2.2", "resolved": "https://registry.npmjs.org/p-limit/-/p-limit-2.2.2.tgz", "integrity": "sha512-WGR+xHecKTr7EbUEhyLSh5Dube9JtdiG78ufaeLxTgpudf/20KqyMioIUZJAezlTIi6evxuoUs9YXc11cU+yzQ==", + "dev": true, "requires": { "p-try": "^2.0.0" } @@ -3821,7 +4094,8 @@ "p-try": { "version": "2.2.0", "resolved": "https://registry.npmjs.org/p-try/-/p-try-2.2.0.tgz", - "integrity": "sha512-R4nPAVTAU0B9D35/Gk3uJf/7XYbQcyohSKdvAxIRSNghFl4e71hVoGnBNQz9cWaXxO2I10KTC+3jMdvvoKw6dQ==" + "integrity": "sha512-R4nPAVTAU0B9D35/Gk3uJf/7XYbQcyohSKdvAxIRSNghFl4e71hVoGnBNQz9cWaXxO2I10KTC+3jMdvvoKw6dQ==", + "dev": true }, "parent-module": { "version": "1.0.1", @@ -3832,11 +4106,6 @@ "callsites": "^3.0.0" } }, - "parse-duration": { - "version": "0.1.2", - "resolved": "https://registry.npmjs.org/parse-duration/-/parse-duration-0.1.2.tgz", - "integrity": "sha512-0qfMZyjOUFBeEIvJ5EayfXJqaEXxQ+Oj2b7tWJM3hvEXvXsYCk05EDVI23oYnEw2NaFYUWdABEVPBvBMh8L/pA==" - }, "parse-json": { "version": "2.2.0", "resolved": "https://registry.npmjs.org/parse-json/-/parse-json-2.2.0.tgz", @@ -3911,11 +4180,6 @@ "resolved": "https://registry.npmjs.org/performance-now/-/performance-now-2.1.0.tgz", "integrity": "sha512-7EAHlyLHI56VEIdK57uwHdHKIaAGbnXPiw0yWbarQZOKaKpvUIgW0jWRVLiatnM+XXlSwsanIBH/hzGMJulMow==" }, - "pify": { - "version": "4.0.1", - "resolved": "https://registry.npmjs.org/pify/-/pify-4.0.1.tgz", - "integrity": "sha512-uB80kBFb/tfd68bVleG9T5GGsGPjJrLAUpR5PZIrhBnIaRTQRjqdJSsIKkOP6OAIFbj7GOrcudc5pNjZ+geV2g==" - }, "pkg-dir": { "version": "2.0.0", "resolved": "https://registry.npmjs.org/pkg-dir/-/pkg-dir-2.0.0.tgz", @@ -3930,6 +4194,68 @@ "resolved": "https://registry.npmjs.org/policyfile/-/policyfile-0.0.4.tgz", "integrity": "sha1-1rgurZiueeviKOLa9ZAzEeyYLk0=" }, + "pprof": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/pprof/-/pprof-3.0.0.tgz", + "integrity": "sha512-uPWbAhoH/zvq1kM3/Fd/wshb4D7sLlGap8t6uCTER4aZRWqqyPYgXzpjWbT0Unn5U25pEy2VREUu27nQ9o9VPA==", + "requires": { + "bindings": "^1.2.1", + "delay": "^4.0.1", + "findit2": "^2.2.3", + "nan": "^2.14.0", + "node-pre-gyp": "^0.16.0", + "p-limit": "^3.0.0", + "pify": "^5.0.0", + "protobufjs": "~6.10.0", + "source-map": "^0.7.3", + "split": "^1.0.1" + }, + "dependencies": { + "nan": { + "version": "2.14.2", + "resolved": "https://registry.npmjs.org/nan/-/nan-2.14.2.tgz", + "integrity": "sha512-M2ufzIiINKCuDfBSAUr1vWQ+vuVcA9kqx8JJUsbQi6yf1uGRyb7HfpdfUr5qLXf3B/t8dPvcjhKMmlfnP47EzQ==" + }, + "p-limit": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/p-limit/-/p-limit-3.1.0.tgz", + "integrity": "sha512-TYOanM3wGwNGsZN2cVTYPArw454xnXj5qmWF1bEoAc4+cU/ol7GVh7odevjp1FNHduHc3KZMcFduxU5Xc6uJRQ==", + "requires": { + "yocto-queue": "^0.1.0" + } + }, + "pify": { + "version": "5.0.0", + "resolved": "https://registry.npmjs.org/pify/-/pify-5.0.0.tgz", + "integrity": "sha512-eW/gHNMlxdSP6dmG6uJip6FXN0EQBwm2clYYd8Wul42Cwu/DK8HEftzsapcNdYe2MfLiIwZqsDk2RDEsTE79hA==" + }, + "protobufjs": { + "version": "6.10.2", + "resolved": "https://registry.npmjs.org/protobufjs/-/protobufjs-6.10.2.tgz", + "integrity": "sha512-27yj+04uF6ya9l+qfpH187aqEzfCF4+Uit0I9ZBQVqK09hk/SQzKa2MUqUpXaVa7LOFRg1TSSr3lVxGOk6c0SQ==", + "requires": { + "@protobufjs/aspromise": "^1.1.2", + "@protobufjs/base64": "^1.1.2", + "@protobufjs/codegen": "^2.0.4", + "@protobufjs/eventemitter": "^1.1.0", + "@protobufjs/fetch": "^1.1.0", + "@protobufjs/float": "^1.0.2", + "@protobufjs/inquire": "^1.1.0", + "@protobufjs/path": "^1.1.2", + "@protobufjs/pool": "^1.1.0", + "@protobufjs/utf8": "^1.1.0", + "@types/long": "^4.0.1", + "@types/node": "^13.7.0", + "long": "^4.0.0" + } + }, + "source-map": { + "version": "0.7.3", + "resolved": "https://registry.npmjs.org/source-map/-/source-map-0.7.3.tgz", + "integrity": "sha512-CkCj6giN3S+n9qrYiBTX5gystlENnRW5jZeNLHpe6aue+SrHcG5VYwujhW9s4dY31mEGsxBDrHR6oI69fTXsaQ==" + } + } + }, "prelude-ls": { "version": "1.1.2", "resolved": "https://registry.npmjs.org/prelude-ls/-/prelude-ls-1.1.2.tgz", @@ -4551,14 +4877,6 @@ } } }, - "pretty-ms": { - "version": "4.0.0", - "resolved": "https://registry.npmjs.org/pretty-ms/-/pretty-ms-4.0.0.tgz", - "integrity": "sha512-qG66ahoLCwpLXD09ZPHSCbUWYTqdosB7SMP4OffgTgL2PBKXMuUsrk5Bwg8q4qPkjTXsKBMr+YK3Ltd/6F9s/Q==", - "requires": { - "parse-ms": "^2.0.0" - } - }, "process-nextick-args": { "version": "2.0.1", "resolved": "https://registry.npmjs.org/process-nextick-args/-/process-nextick-args-2.0.1.tgz", @@ -4723,6 +5041,29 @@ "unpipe": "1.0.0" } }, + "rc": { + "version": "1.2.8", + "resolved": "https://registry.npmjs.org/rc/-/rc-1.2.8.tgz", + "integrity": "sha512-y3bGgqKj3QBdxLbLkomlohkvsA8gdAiUQlSBJnBhfn+BPxg4bc62d8TcBW15wavDfgexCgccckhcZvywyQYPOw==", + "requires": { + "deep-extend": "^0.6.0", + "ini": "~1.3.0", + "minimist": "^1.2.0", + "strip-json-comments": "~2.0.1" + }, + "dependencies": { + "minimist": { + "version": "1.2.5", + "resolved": "https://registry.npmjs.org/minimist/-/minimist-1.2.5.tgz", + "integrity": "sha512-FM9nNUYrRBAELZQT3xeZQ7fmMOBg6nWNmJKTcgsJeaLstP/UODVpGsr5OhXhhXg6f+qtJ8uiZ+PUxkDWcgIXLw==" + }, + "strip-json-comments": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/strip-json-comments/-/strip-json-comments-2.0.1.tgz", + "integrity": "sha1-PFMZQukIwml8DsNEhYwobHygpgo=" + } + } + }, "read-pkg": { "version": "2.0.0", "resolved": "https://registry.npmjs.org/read-pkg/-/read-pkg-2.0.0.tgz", @@ -4859,31 +5200,6 @@ "integrity": "sha512-fGxEI7+wsG9xrvdjsrlmL22OMTTiHRwAMroiEeMgq8gzoLC/PQr7RsRDSTLUg/bZAZtF+TVIkHc6/4RIKrui+Q==", "dev": true }, - "require-in-the-middle": { - "version": "4.0.1", - "resolved": "https://registry.npmjs.org/require-in-the-middle/-/require-in-the-middle-4.0.1.tgz", - "integrity": "sha512-EfkM2zANyGkrfIExsECMeNn/uzjvHrE9h36yLXSavmrDiH4tgDNvltAmEKnt4PNLbqKPHZz+uszW2wTKrLUX0w==", - "requires": { - "debug": "^4.1.1", - "module-details-from-path": "^1.0.3", - "resolve": "^1.12.0" - }, - "dependencies": { - "debug": { - "version": "4.1.1", - "resolved": "https://registry.npmjs.org/debug/-/debug-4.1.1.tgz", - "integrity": "sha512-pYAIzeRo8J6KPEaJ0VWOh5Pzkbw/RetuzehGM7QRRX5he4fPHx2rdKMB256ehJCkX+XRQm16eZLqLNS8RSZXZw==", - "requires": { - "ms": "^2.1.1" - } - }, - "ms": { - "version": "2.1.2", - "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.2.tgz", - "integrity": "sha512-sGkPx+VjMtmA6MX27oA4FBFELFCZZ4S4XqeGOXCv68tT+jb3vk/RyaKWP0PTKyWtmLSM0b+adUTEvbs1PEaH2w==" - } - } - }, "require-like": { "version": "0.1.2", "resolved": "https://registry.npmjs.org/require-like/-/require-like-0.1.2.tgz", @@ -4926,11 +5242,6 @@ "signal-exit": "^3.0.2" } }, - "retry-axios": { - "version": "0.3.2", - "resolved": "https://registry.npmjs.org/retry-axios/-/retry-axios-0.3.2.tgz", - "integrity": "sha512-jp4YlI0qyDFfXiXGhkCOliBN1G7fRH03Nqy8YdShzGqbY5/9S2x/IR6C88ls2DFkbWuL3ASkP7QD3pVrNpPgwQ==" - }, "retry-request": { "version": "4.1.1", "resolved": "https://registry.npmjs.org/retry-request/-/retry-request-4.1.1.tgz", @@ -5019,6 +5330,11 @@ } } }, + "sax": { + "version": "1.2.4", + "resolved": "https://registry.npmjs.org/sax/-/sax-1.2.4.tgz", + "integrity": "sha512-NqVDv9TpANUjFm0N8uM5GxL36UgKi9/atZw+x7YFnQ8ckwFGKrl4xX4yWtrey3UJm5nP1kUbnYgLopqWNSRhWw==" + }, "semver": { "version": "6.3.0", "resolved": "https://registry.npmjs.org/semver/-/semver-6.3.0.tgz", @@ -5065,8 +5381,7 @@ "set-blocking": { "version": "2.0.0", "resolved": "https://registry.npmjs.org/set-blocking/-/set-blocking-2.0.0.tgz", - "integrity": "sha512-KiKBS8AnWGEyLzofFfmvKwpdPzqiy16LvQfK3yv/fVH7Bj13/wl3JSR1J+rfgRE9q7xUJK4qvgS8raSOeLUehw==", - "dev": true + "integrity": "sha512-KiKBS8AnWGEyLzofFfmvKwpdPzqiy16LvQfK3yv/fVH7Bj13/wl3JSR1J+rfgRE9q7xUJK4qvgS8raSOeLUehw==" }, "setprototypeof": { "version": "1.1.1", @@ -5111,8 +5426,7 @@ "signal-exit": { "version": "3.0.3", "resolved": "https://registry.npmjs.org/signal-exit/-/signal-exit-3.0.3.tgz", - "integrity": "sha512-VUJ49FC8U1OxwZLxIbTTrDvLnf/6TDgxZcK8wxR8zs13xpx7xbG60ndBlhNrFi2EMuFRoeDoJO7wthSLq42EjA==", - "dev": true + "integrity": "sha512-VUJ49FC8U1OxwZLxIbTTrDvLnf/6TDgxZcK8wxR8zs13xpx7xbG60ndBlhNrFi2EMuFRoeDoJO7wthSLq42EjA==" }, "sinon": { "version": "2.4.1", @@ -5214,6 +5528,15 @@ "resolved": "https://registry.npmjs.org/source-map/-/source-map-0.6.1.tgz", "integrity": "sha512-UjgapumWlbMhkBgzT7Ykc5YXUT46F0iKu8SGXq0bcwP5dz/h0Plj6enJqjz1Zbq2l5WaqYnrVbwWOWMyF3F47g==" }, + "source-map-support": { + "version": "0.5.19", + "resolved": "https://registry.npmjs.org/source-map-support/-/source-map-support-0.5.19.tgz", + "integrity": "sha512-Wonm7zOCIJzBGQdB+thsPar0kYuCIzYvxZwlBa87yi/Mdjv7Tip2cyVbLj5o0cFPN4EVkuTwb3GDDyUx2DGnGw==", + "requires": { + "buffer-from": "^1.0.0", + "source-map": "^0.6.0" + } + }, "spdx-correct": { "version": "3.1.1", "resolved": "https://registry.npmjs.org/spdx-correct/-/spdx-correct-3.1.1.tgz", @@ -5286,11 +5609,6 @@ "resolved": "https://registry.npmjs.org/standard-as-callback/-/standard-as-callback-2.0.1.tgz", "integrity": "sha512-NQOxSeB8gOI5WjSaxjBgog2QFw55FV8TkS6Y07BiB3VJ8xNTvUYm0wl0s8ObgQ5NhdpnNfigMIKjgPESzgr4tg==" }, - "statsd-parser": { - "version": "0.0.4", - "resolved": "https://registry.npmjs.org/statsd-parser/-/statsd-parser-0.0.4.tgz", - "integrity": "sha512-7XO+ur89EalMXXFQaydsczB8sclr5nDsNIoUu0IzJx1pIbHUhO3LtpSzBwetIuU9DyTLMiVaJBMtWS/Nb2KR4g==" - }, "statuses": { "version": "1.5.0", "resolved": "https://registry.npmjs.org/statuses/-/statuses-1.5.0.tgz", @@ -5439,6 +5757,27 @@ } } }, + "tar": { + "version": "4.4.13", + "resolved": "https://registry.npmjs.org/tar/-/tar-4.4.13.tgz", + "integrity": "sha512-w2VwSrBoHa5BsSyH+KxEqeQBAllHhccyMFVHtGtdMpF4W7IRWfZjFiQceJPChOeTsSDVUpER2T8FA93pr0L+QA==", + "requires": { + "chownr": "^1.1.1", + "fs-minipass": "^1.2.5", + "minipass": "^2.8.6", + "minizlib": "^1.2.1", + "mkdirp": "^0.5.0", + "safe-buffer": "^5.1.2", + "yallist": "^3.0.3" + }, + "dependencies": { + "safe-buffer": { + "version": "5.2.1", + "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.2.1.tgz", + "integrity": "sha512-rp3So07KcdmmKbGvgaNxQSJr7bGVSVk5S9Eq1F+ppbRo70+YeaDxkw5Dd8NPN+GD6bjnYm2VuPuCXmpuYvmCXQ==" + } + } + }, "tdigest": { "version": "0.1.1", "resolved": "https://registry.npmjs.org/tdigest/-/tdigest-0.1.1.tgz", @@ -5447,23 +5786,6 @@ "bintrees": "1.0.1" } }, - "teeny-request": { - "version": "3.11.3", - "resolved": "https://registry.npmjs.org/teeny-request/-/teeny-request-3.11.3.tgz", - "integrity": "sha512-CKncqSF7sH6p4rzCgkb/z/Pcos5efl0DmolzvlqRQUNcpRIruOhY9+T1FsIlyEbfWd7MsFpodROOwHYh2BaXzw==", - "requires": { - "https-proxy-agent": "^2.2.1", - "node-fetch": "^2.2.0", - "uuid": "^3.3.2" - }, - "dependencies": { - "uuid": { - "version": "3.4.0", - "resolved": "https://registry.npmjs.org/uuid/-/uuid-3.4.0.tgz", - "integrity": "sha512-HjSDRw6gZE5JMggctHBcjVak08+KEVhSIiDzFnT9S9aegmp85S/bReBVTb4QTFaRNptJ9kuYaNhnbNEOkbKb/A==" - } - } - }, "text-encoding": { "version": "0.6.4", "resolved": "https://registry.npmjs.org/text-encoding/-/text-encoding-0.6.4.tgz", @@ -5646,7 +5968,7 @@ "underscore": { "version": "1.7.0", "resolved": "https://registry.npmjs.org/underscore/-/underscore-1.7.0.tgz", - "integrity": "sha512-cp0oQQyZhUM1kpJDLdGO1jPZHgS/MpzoWYfe9+CM2h/QGDZlqwT2T3YGukuBdaNJ/CAPoeyAZRRHz8JFo176vA==" + "integrity": "sha1-a7rwh3UA02vjTsqlhODbn+8DUgk=" }, "unpipe": { "version": "1.0.0", @@ -5801,6 +6123,43 @@ "integrity": "sha512-B+enWhmw6cjfVC7kS8Pj9pCrKSc5txArRyaYGe088shv/FGWH+0Rjx/xPgtsWfsUtS27FkP697E4DDhgrgoc0Q==", "dev": true }, + "wide-align": { + "version": "1.1.3", + "resolved": "https://registry.npmjs.org/wide-align/-/wide-align-1.1.3.tgz", + "integrity": "sha512-QGkOQc8XL6Bt5PwnsExKBPuMKBxnGxWWW3fU55Xt4feHozMUhdUMaBCk290qpm/wG5u/RSKzwdAC4i51YigihA==", + "requires": { + "string-width": "^1.0.2 || 2" + }, + "dependencies": { + "ansi-regex": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-3.0.0.tgz", + "integrity": "sha1-7QMXwyIGT3lGbAKWa922Bas32Zg=" + }, + "is-fullwidth-code-point": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/is-fullwidth-code-point/-/is-fullwidth-code-point-2.0.0.tgz", + "integrity": "sha1-o7MKXE8ZkYMWeqq5O+764937ZU8=" + }, + "string-width": { + "version": "2.1.1", + "resolved": "https://registry.npmjs.org/string-width/-/string-width-2.1.1.tgz", + "integrity": "sha512-nOqH59deCq9SRHlxq1Aw85Jnt4w6KvLKqWVik6oA9ZklXLNIOlqg4F2yrT1MVaTjAqvVwdfeZ7w7aCvJD7ugkw==", + "requires": { + "is-fullwidth-code-point": "^2.0.0", + "strip-ansi": "^4.0.0" + } + }, + "strip-ansi": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-4.0.0.tgz", + "integrity": "sha1-qEeQIusaw2iocTibY1JixQXuNo8=", + "requires": { + "ansi-regex": "^3.0.0" + } + } + } + }, "word-wrap": { "version": "1.2.3", "resolved": "https://registry.npmjs.org/word-wrap/-/word-wrap-1.2.3.tgz", @@ -5977,6 +6336,11 @@ "resolved": "https://registry.npmjs.org/yn/-/yn-3.1.1.tgz", "integrity": "sha512-Ux4ygGWsu2c7isFWe8Yu1YluJmqVhxqK2cLXNQA5AcC3QfbGNpM7fu0Y8b/z16pXLnFxZYvWhd3fhBY9DLmC6Q==" }, + "yocto-queue": { + "version": "0.1.0", + "resolved": "https://registry.npmjs.org/yocto-queue/-/yocto-queue-0.1.0.tgz", + "integrity": "sha512-rVksvsnNCdJ/ohGc6xgPwyN8eheCxsiLM8mxuE/t/mOVqJewPuO1miLpTHQiRgTKCLexL4MeAFVagts7HmNZ2Q==" + }, "zeparser": { "version": "0.0.5", "resolved": "https://registry.npmjs.org/zeparser/-/zeparser-0.0.5.tgz", diff --git a/services/real-time/package.json b/services/real-time/package.json index 89012d6ed6..a806d93ab6 100644 --- a/services/real-time/package.json +++ b/services/real-time/package.json @@ -19,6 +19,7 @@ "format:fix": "node_modules/.bin/prettier-eslint $PWD'/**/*.js' --write" }, "dependencies": { + "@overleaf/metrics": "^3.4.1", "@overleaf/o-error": "^3.1.0", "@overleaf/redis-wrapper": "^2.0.0", "async": "^0.9.0", @@ -30,7 +31,6 @@ "express": "^4.17.1", "express-session": "^1.17.1", "logger-sharelatex": "^2.2.0", - "metrics-sharelatex": "^2.6.2", "request": "^2.88.2", "settings-sharelatex": "^1.1.0", "socket.io": "https://github.com/overleaf/socket.io/archive/0.9.19-overleaf-4.tar.gz", diff --git a/services/real-time/test/unit/js/ChannelManagerTests.js b/services/real-time/test/unit/js/ChannelManagerTests.js index 95e9000a49..f76a109129 100644 --- a/services/real-time/test/unit/js/ChannelManagerTests.js +++ b/services/real-time/test/unit/js/ChannelManagerTests.js @@ -23,7 +23,7 @@ describe('ChannelManager', function () { return (this.ChannelManager = SandboxedModule.require(modulePath, { requires: { 'settings-sharelatex': (this.settings = {}), - 'metrics-sharelatex': (this.metrics = { + '@overleaf/metrics': (this.metrics = { inc: sinon.stub(), summary: sinon.stub() }), diff --git a/services/real-time/test/unit/js/DocumentUpdaterControllerTests.js b/services/real-time/test/unit/js/DocumentUpdaterControllerTests.js index d42505da7e..832ad9490a 100644 --- a/services/real-time/test/unit/js/DocumentUpdaterControllerTests.js +++ b/services/real-time/test/unit/js/DocumentUpdaterControllerTests.js @@ -58,7 +58,7 @@ describe('DocumentUpdaterController', function () { }), './EventLogger': (this.EventLogger = { checkEventOrder: sinon.stub() }), './HealthCheckManager': { check: sinon.stub() }, - 'metrics-sharelatex': (this.metrics = { inc: sinon.stub() }), + '@overleaf/metrics': (this.metrics = { inc: sinon.stub() }), './RoomManager': (this.RoomManager = { eventSource: sinon.stub().returns(this.RoomEvents) }), diff --git a/services/real-time/test/unit/js/DocumentUpdaterManagerTests.js b/services/real-time/test/unit/js/DocumentUpdaterManagerTests.js index 97a032b76f..d6da97ddab 100644 --- a/services/real-time/test/unit/js/DocumentUpdaterManagerTests.js +++ b/services/real-time/test/unit/js/DocumentUpdaterManagerTests.js @@ -48,7 +48,7 @@ describe('DocumentUpdaterManager', function () { }), request: (this.request = {}), '@overleaf/redis-wrapper': { createClient: () => this.rclient }, - 'metrics-sharelatex': (this.Metrics = { + '@overleaf/metrics': (this.Metrics = { summary: sinon.stub(), Timer: (Timer = class Timer { done() {} diff --git a/services/real-time/test/unit/js/EventLoggerTests.js b/services/real-time/test/unit/js/EventLoggerTests.js index 7152f92ce7..c95d65ae2e 100644 --- a/services/real-time/test/unit/js/EventLoggerTests.js +++ b/services/real-time/test/unit/js/EventLoggerTests.js @@ -25,7 +25,7 @@ describe('EventLogger', function () { error: sinon.stub(), warn: sinon.stub() }), - 'metrics-sharelatex': (this.metrics = { inc: sinon.stub() }) + '@overleaf/metrics': (this.metrics = { inc: sinon.stub() }) } }) this.channel = 'applied-ops' diff --git a/services/real-time/test/unit/js/RoomManagerTests.js b/services/real-time/test/unit/js/RoomManagerTests.js index a202720ec7..84aeb7a868 100644 --- a/services/real-time/test/unit/js/RoomManagerTests.js +++ b/services/real-time/test/unit/js/RoomManagerTests.js @@ -31,7 +31,7 @@ describe('RoomManager', function () { warn: sinon.stub(), error: sinon.stub() }), - 'metrics-sharelatex': (this.metrics = { gauge: sinon.stub() }) + '@overleaf/metrics': (this.metrics = { gauge: sinon.stub() }) } }) this.RoomManager._clientsInRoom = sinon.stub() diff --git a/services/real-time/test/unit/js/WebsocketControllerTests.js b/services/real-time/test/unit/js/WebsocketControllerTests.js index f210a16ec8..c5f4a1ac42 100644 --- a/services/real-time/test/unit/js/WebsocketControllerTests.js +++ b/services/real-time/test/unit/js/WebsocketControllerTests.js @@ -55,7 +55,7 @@ describe('WebsocketController', function () { error: sinon.stub(), warn: sinon.stub() }), - 'metrics-sharelatex': (this.metrics = { + '@overleaf/metrics': (this.metrics = { inc: sinon.stub(), set: sinon.stub() }), From 1ddcdf38ca15e149ca4934f24b8fdf0337d1bbcf Mon Sep 17 00:00:00 2001 From: Brian Gough Date: Fri, 27 Nov 2020 14:42:42 +0000 Subject: [PATCH 455/491] add comment about health checks on '/' --- services/real-time/app.js | 1 + 1 file changed, 1 insertion(+) diff --git a/services/real-time/app.js b/services/real-time/app.js index 7d7d85c83f..e65561cf43 100644 --- a/services/real-time/app.js +++ b/services/real-time/app.js @@ -78,6 +78,7 @@ io.configure(function () { ]) }) +// a 200 response on '/' is required for load balancer health checks app.get('/', (req, res) => res.send('real-time-sharelatex is alive')) app.get('/status', function (req, res) { From 8fec6723d80af8996665d9b40e0db7009a4b4296 Mon Sep 17 00:00:00 2001 From: John Lees-Miller Date: Thu, 3 Dec 2020 10:27:38 +0000 Subject: [PATCH 456/491] Add correct license --- services/real-time/LICENSE | 661 ++++++++++++++++++++++++++++++++ services/real-time/package.json | 2 + 2 files changed, 663 insertions(+) create mode 100644 services/real-time/LICENSE diff --git a/services/real-time/LICENSE b/services/real-time/LICENSE new file mode 100644 index 0000000000..dba13ed2dd --- /dev/null +++ b/services/real-time/LICENSE @@ -0,0 +1,661 @@ + GNU AFFERO GENERAL PUBLIC LICENSE + Version 3, 19 November 2007 + + Copyright (C) 2007 Free Software Foundation, Inc. + Everyone is permitted to copy and distribute verbatim copies + of this license document, but changing it is not allowed. + + Preamble + + The GNU Affero General Public License is a free, copyleft license for +software and other kinds of works, specifically designed to ensure +cooperation with the community in the case of network server software. + + The licenses for most software and other practical works are designed +to take away your freedom to share and change the works. By contrast, +our General Public Licenses are intended to guarantee your freedom to +share and change all versions of a program--to make sure it remains free +software for all its users. + + When we speak of free software, we are referring to freedom, not +price. Our General Public Licenses are designed to make sure that you +have the freedom to distribute copies of free software (and charge for +them if you wish), that you receive source code or can get it if you +want it, that you can change the software or use pieces of it in new +free programs, and that you know you can do these things. + + Developers that use our General Public Licenses protect your rights +with two steps: (1) assert copyright on the software, and (2) offer +you this License which gives you legal permission to copy, distribute +and/or modify the software. + + A secondary benefit of defending all users' freedom is that +improvements made in alternate versions of the program, if they +receive widespread use, become available for other developers to +incorporate. Many developers of free software are heartened and +encouraged by the resulting cooperation. However, in the case of +software used on network servers, this result may fail to come about. +The GNU General Public License permits making a modified version and +letting the public access it on a server without ever releasing its +source code to the public. + + The GNU Affero General Public License is designed specifically to +ensure that, in such cases, the modified source code becomes available +to the community. It requires the operator of a network server to +provide the source code of the modified version running there to the +users of that server. Therefore, public use of a modified version, on +a publicly accessible server, gives the public access to the source +code of the modified version. + + An older license, called the Affero General Public License and +published by Affero, was designed to accomplish similar goals. This is +a different license, not a version of the Affero GPL, but Affero has +released a new version of the Affero GPL which permits relicensing under +this license. + + The precise terms and conditions for copying, distribution and +modification follow. + + TERMS AND CONDITIONS + + 0. Definitions. + + "This License" refers to version 3 of the GNU Affero General Public License. + + "Copyright" also means copyright-like laws that apply to other kinds of +works, such as semiconductor masks. + + "The Program" refers to any copyrightable work licensed under this +License. Each licensee is addressed as "you". "Licensees" and +"recipients" may be individuals or organizations. + + To "modify" a work means to copy from or adapt all or part of the work +in a fashion requiring copyright permission, other than the making of an +exact copy. The resulting work is called a "modified version" of the +earlier work or a work "based on" the earlier work. + + A "covered work" means either the unmodified Program or a work based +on the Program. + + To "propagate" a work means to do anything with it that, without +permission, would make you directly or secondarily liable for +infringement under applicable copyright law, except executing it on a +computer or modifying a private copy. Propagation includes copying, +distribution (with or without modification), making available to the +public, and in some countries other activities as well. + + To "convey" a work means any kind of propagation that enables other +parties to make or receive copies. Mere interaction with a user through +a computer network, with no transfer of a copy, is not conveying. + + An interactive user interface displays "Appropriate Legal Notices" +to the extent that it includes a convenient and prominently visible +feature that (1) displays an appropriate copyright notice, and (2) +tells the user that there is no warranty for the work (except to the +extent that warranties are provided), that licensees may convey the +work under this License, and how to view a copy of this License. If +the interface presents a list of user commands or options, such as a +menu, a prominent item in the list meets this criterion. + + 1. Source Code. + + The "source code" for a work means the preferred form of the work +for making modifications to it. "Object code" means any non-source +form of a work. + + A "Standard Interface" means an interface that either is an official +standard defined by a recognized standards body, or, in the case of +interfaces specified for a particular programming language, one that +is widely used among developers working in that language. + + The "System Libraries" of an executable work include anything, other +than the work as a whole, that (a) is included in the normal form of +packaging a Major Component, but which is not part of that Major +Component, and (b) serves only to enable use of the work with that +Major Component, or to implement a Standard Interface for which an +implementation is available to the public in source code form. A +"Major Component", in this context, means a major essential component +(kernel, window system, and so on) of the specific operating system +(if any) on which the executable work runs, or a compiler used to +produce the work, or an object code interpreter used to run it. + + The "Corresponding Source" for a work in object code form means all +the source code needed to generate, install, and (for an executable +work) run the object code and to modify the work, including scripts to +control those activities. However, it does not include the work's +System Libraries, or general-purpose tools or generally available free +programs which are used unmodified in performing those activities but +which are not part of the work. For example, Corresponding Source +includes interface definition files associated with source files for +the work, and the source code for shared libraries and dynamically +linked subprograms that the work is specifically designed to require, +such as by intimate data communication or control flow between those +subprograms and other parts of the work. + + The Corresponding Source need not include anything that users +can regenerate automatically from other parts of the Corresponding +Source. + + The Corresponding Source for a work in source code form is that +same work. + + 2. Basic Permissions. + + All rights granted under this License are granted for the term of +copyright on the Program, and are irrevocable provided the stated +conditions are met. This License explicitly affirms your unlimited +permission to run the unmodified Program. The output from running a +covered work is covered by this License only if the output, given its +content, constitutes a covered work. This License acknowledges your +rights of fair use or other equivalent, as provided by copyright law. + + You may make, run and propagate covered works that you do not +convey, without conditions so long as your license otherwise remains +in force. You may convey covered works to others for the sole purpose +of having them make modifications exclusively for you, or provide you +with facilities for running those works, provided that you comply with +the terms of this License in conveying all material for which you do +not control copyright. Those thus making or running the covered works +for you must do so exclusively on your behalf, under your direction +and control, on terms that prohibit them from making any copies of +your copyrighted material outside their relationship with you. + + Conveying under any other circumstances is permitted solely under +the conditions stated below. Sublicensing is not allowed; section 10 +makes it unnecessary. + + 3. Protecting Users' Legal Rights From Anti-Circumvention Law. + + No covered work shall be deemed part of an effective technological +measure under any applicable law fulfilling obligations under article +11 of the WIPO copyright treaty adopted on 20 December 1996, or +similar laws prohibiting or restricting circumvention of such +measures. + + When you convey a covered work, you waive any legal power to forbid +circumvention of technological measures to the extent such circumvention +is effected by exercising rights under this License with respect to +the covered work, and you disclaim any intention to limit operation or +modification of the work as a means of enforcing, against the work's +users, your or third parties' legal rights to forbid circumvention of +technological measures. + + 4. Conveying Verbatim Copies. + + You may convey verbatim copies of the Program's source code as you +receive it, in any medium, provided that you conspicuously and +appropriately publish on each copy an appropriate copyright notice; +keep intact all notices stating that this License and any +non-permissive terms added in accord with section 7 apply to the code; +keep intact all notices of the absence of any warranty; and give all +recipients a copy of this License along with the Program. + + You may charge any price or no price for each copy that you convey, +and you may offer support or warranty protection for a fee. + + 5. Conveying Modified Source Versions. + + You may convey a work based on the Program, or the modifications to +produce it from the Program, in the form of source code under the +terms of section 4, provided that you also meet all of these conditions: + + a) The work must carry prominent notices stating that you modified + it, and giving a relevant date. + + b) The work must carry prominent notices stating that it is + released under this License and any conditions added under section + 7. This requirement modifies the requirement in section 4 to + "keep intact all notices". + + c) You must license the entire work, as a whole, under this + License to anyone who comes into possession of a copy. This + License will therefore apply, along with any applicable section 7 + additional terms, to the whole of the work, and all its parts, + regardless of how they are packaged. This License gives no + permission to license the work in any other way, but it does not + invalidate such permission if you have separately received it. + + d) If the work has interactive user interfaces, each must display + Appropriate Legal Notices; however, if the Program has interactive + interfaces that do not display Appropriate Legal Notices, your + work need not make them do so. + + A compilation of a covered work with other separate and independent +works, which are not by their nature extensions of the covered work, +and which are not combined with it such as to form a larger program, +in or on a volume of a storage or distribution medium, is called an +"aggregate" if the compilation and its resulting copyright are not +used to limit the access or legal rights of the compilation's users +beyond what the individual works permit. Inclusion of a covered work +in an aggregate does not cause this License to apply to the other +parts of the aggregate. + + 6. Conveying Non-Source Forms. + + You may convey a covered work in object code form under the terms +of sections 4 and 5, provided that you also convey the +machine-readable Corresponding Source under the terms of this License, +in one of these ways: + + a) Convey the object code in, or embodied in, a physical product + (including a physical distribution medium), accompanied by the + Corresponding Source fixed on a durable physical medium + customarily used for software interchange. + + b) Convey the object code in, or embodied in, a physical product + (including a physical distribution medium), accompanied by a + written offer, valid for at least three years and valid for as + long as you offer spare parts or customer support for that product + model, to give anyone who possesses the object code either (1) a + copy of the Corresponding Source for all the software in the + product that is covered by this License, on a durable physical + medium customarily used for software interchange, for a price no + more than your reasonable cost of physically performing this + conveying of source, or (2) access to copy the + Corresponding Source from a network server at no charge. + + c) Convey individual copies of the object code with a copy of the + written offer to provide the Corresponding Source. This + alternative is allowed only occasionally and noncommercially, and + only if you received the object code with such an offer, in accord + with subsection 6b. + + d) Convey the object code by offering access from a designated + place (gratis or for a charge), and offer equivalent access to the + Corresponding Source in the same way through the same place at no + further charge. You need not require recipients to copy the + Corresponding Source along with the object code. If the place to + copy the object code is a network server, the Corresponding Source + may be on a different server (operated by you or a third party) + that supports equivalent copying facilities, provided you maintain + clear directions next to the object code saying where to find the + Corresponding Source. Regardless of what server hosts the + Corresponding Source, you remain obligated to ensure that it is + available for as long as needed to satisfy these requirements. + + e) Convey the object code using peer-to-peer transmission, provided + you inform other peers where the object code and Corresponding + Source of the work are being offered to the general public at no + charge under subsection 6d. + + A separable portion of the object code, whose source code is excluded +from the Corresponding Source as a System Library, need not be +included in conveying the object code work. + + A "User Product" is either (1) a "consumer product", which means any +tangible personal property which is normally used for personal, family, +or household purposes, or (2) anything designed or sold for incorporation +into a dwelling. In determining whether a product is a consumer product, +doubtful cases shall be resolved in favor of coverage. For a particular +product received by a particular user, "normally used" refers to a +typical or common use of that class of product, regardless of the status +of the particular user or of the way in which the particular user +actually uses, or expects or is expected to use, the product. A product +is a consumer product regardless of whether the product has substantial +commercial, industrial or non-consumer uses, unless such uses represent +the only significant mode of use of the product. + + "Installation Information" for a User Product means any methods, +procedures, authorization keys, or other information required to install +and execute modified versions of a covered work in that User Product from +a modified version of its Corresponding Source. The information must +suffice to ensure that the continued functioning of the modified object +code is in no case prevented or interfered with solely because +modification has been made. + + If you convey an object code work under this section in, or with, or +specifically for use in, a User Product, and the conveying occurs as +part of a transaction in which the right of possession and use of the +User Product is transferred to the recipient in perpetuity or for a +fixed term (regardless of how the transaction is characterized), the +Corresponding Source conveyed under this section must be accompanied +by the Installation Information. But this requirement does not apply +if neither you nor any third party retains the ability to install +modified object code on the User Product (for example, the work has +been installed in ROM). + + The requirement to provide Installation Information does not include a +requirement to continue to provide support service, warranty, or updates +for a work that has been modified or installed by the recipient, or for +the User Product in which it has been modified or installed. Access to a +network may be denied when the modification itself materially and +adversely affects the operation of the network or violates the rules and +protocols for communication across the network. + + Corresponding Source conveyed, and Installation Information provided, +in accord with this section must be in a format that is publicly +documented (and with an implementation available to the public in +source code form), and must require no special password or key for +unpacking, reading or copying. + + 7. Additional Terms. + + "Additional permissions" are terms that supplement the terms of this +License by making exceptions from one or more of its conditions. +Additional permissions that are applicable to the entire Program shall +be treated as though they were included in this License, to the extent +that they are valid under applicable law. If additional permissions +apply only to part of the Program, that part may be used separately +under those permissions, but the entire Program remains governed by +this License without regard to the additional permissions. + + When you convey a copy of a covered work, you may at your option +remove any additional permissions from that copy, or from any part of +it. (Additional permissions may be written to require their own +removal in certain cases when you modify the work.) You may place +additional permissions on material, added by you to a covered work, +for which you have or can give appropriate copyright permission. + + Notwithstanding any other provision of this License, for material you +add to a covered work, you may (if authorized by the copyright holders of +that material) supplement the terms of this License with terms: + + a) Disclaiming warranty or limiting liability differently from the + terms of sections 15 and 16 of this License; or + + b) Requiring preservation of specified reasonable legal notices or + author attributions in that material or in the Appropriate Legal + Notices displayed by works containing it; or + + c) Prohibiting misrepresentation of the origin of that material, or + requiring that modified versions of such material be marked in + reasonable ways as different from the original version; or + + d) Limiting the use for publicity purposes of names of licensors or + authors of the material; or + + e) Declining to grant rights under trademark law for use of some + trade names, trademarks, or service marks; or + + f) Requiring indemnification of licensors and authors of that + material by anyone who conveys the material (or modified versions of + it) with contractual assumptions of liability to the recipient, for + any liability that these contractual assumptions directly impose on + those licensors and authors. + + All other non-permissive additional terms are considered "further +restrictions" within the meaning of section 10. If the Program as you +received it, or any part of it, contains a notice stating that it is +governed by this License along with a term that is a further +restriction, you may remove that term. If a license document contains +a further restriction but permits relicensing or conveying under this +License, you may add to a covered work material governed by the terms +of that license document, provided that the further restriction does +not survive such relicensing or conveying. + + If you add terms to a covered work in accord with this section, you +must place, in the relevant source files, a statement of the +additional terms that apply to those files, or a notice indicating +where to find the applicable terms. + + Additional terms, permissive or non-permissive, may be stated in the +form of a separately written license, or stated as exceptions; +the above requirements apply either way. + + 8. Termination. + + You may not propagate or modify a covered work except as expressly +provided under this License. Any attempt otherwise to propagate or +modify it is void, and will automatically terminate your rights under +this License (including any patent licenses granted under the third +paragraph of section 11). + + However, if you cease all violation of this License, then your +license from a particular copyright holder is reinstated (a) +provisionally, unless and until the copyright holder explicitly and +finally terminates your license, and (b) permanently, if the copyright +holder fails to notify you of the violation by some reasonable means +prior to 60 days after the cessation. + + Moreover, your license from a particular copyright holder is +reinstated permanently if the copyright holder notifies you of the +violation by some reasonable means, this is the first time you have +received notice of violation of this License (for any work) from that +copyright holder, and you cure the violation prior to 30 days after +your receipt of the notice. + + Termination of your rights under this section does not terminate the +licenses of parties who have received copies or rights from you under +this License. If your rights have been terminated and not permanently +reinstated, you do not qualify to receive new licenses for the same +material under section 10. + + 9. Acceptance Not Required for Having Copies. + + You are not required to accept this License in order to receive or +run a copy of the Program. Ancillary propagation of a covered work +occurring solely as a consequence of using peer-to-peer transmission +to receive a copy likewise does not require acceptance. However, +nothing other than this License grants you permission to propagate or +modify any covered work. These actions infringe copyright if you do +not accept this License. Therefore, by modifying or propagating a +covered work, you indicate your acceptance of this License to do so. + + 10. Automatic Licensing of Downstream Recipients. + + Each time you convey a covered work, the recipient automatically +receives a license from the original licensors, to run, modify and +propagate that work, subject to this License. You are not responsible +for enforcing compliance by third parties with this License. + + An "entity transaction" is a transaction transferring control of an +organization, or substantially all assets of one, or subdividing an +organization, or merging organizations. If propagation of a covered +work results from an entity transaction, each party to that +transaction who receives a copy of the work also receives whatever +licenses to the work the party's predecessor in interest had or could +give under the previous paragraph, plus a right to possession of the +Corresponding Source of the work from the predecessor in interest, if +the predecessor has it or can get it with reasonable efforts. + + You may not impose any further restrictions on the exercise of the +rights granted or affirmed under this License. For example, you may +not impose a license fee, royalty, or other charge for exercise of +rights granted under this License, and you may not initiate litigation +(including a cross-claim or counterclaim in a lawsuit) alleging that +any patent claim is infringed by making, using, selling, offering for +sale, or importing the Program or any portion of it. + + 11. Patents. + + A "contributor" is a copyright holder who authorizes use under this +License of the Program or a work on which the Program is based. The +work thus licensed is called the contributor's "contributor version". + + A contributor's "essential patent claims" are all patent claims +owned or controlled by the contributor, whether already acquired or +hereafter acquired, that would be infringed by some manner, permitted +by this License, of making, using, or selling its contributor version, +but do not include claims that would be infringed only as a +consequence of further modification of the contributor version. For +purposes of this definition, "control" includes the right to grant +patent sublicenses in a manner consistent with the requirements of +this License. + + Each contributor grants you a non-exclusive, worldwide, royalty-free +patent license under the contributor's essential patent claims, to +make, use, sell, offer for sale, import and otherwise run, modify and +propagate the contents of its contributor version. + + In the following three paragraphs, a "patent license" is any express +agreement or commitment, however denominated, not to enforce a patent +(such as an express permission to practice a patent or covenant not to +sue for patent infringement). To "grant" such a patent license to a +party means to make such an agreement or commitment not to enforce a +patent against the party. + + If you convey a covered work, knowingly relying on a patent license, +and the Corresponding Source of the work is not available for anyone +to copy, free of charge and under the terms of this License, through a +publicly available network server or other readily accessible means, +then you must either (1) cause the Corresponding Source to be so +available, or (2) arrange to deprive yourself of the benefit of the +patent license for this particular work, or (3) arrange, in a manner +consistent with the requirements of this License, to extend the patent +license to downstream recipients. "Knowingly relying" means you have +actual knowledge that, but for the patent license, your conveying the +covered work in a country, or your recipient's use of the covered work +in a country, would infringe one or more identifiable patents in that +country that you have reason to believe are valid. + + If, pursuant to or in connection with a single transaction or +arrangement, you convey, or propagate by procuring conveyance of, a +covered work, and grant a patent license to some of the parties +receiving the covered work authorizing them to use, propagate, modify +or convey a specific copy of the covered work, then the patent license +you grant is automatically extended to all recipients of the covered +work and works based on it. + + A patent license is "discriminatory" if it does not include within +the scope of its coverage, prohibits the exercise of, or is +conditioned on the non-exercise of one or more of the rights that are +specifically granted under this License. You may not convey a covered +work if you are a party to an arrangement with a third party that is +in the business of distributing software, under which you make payment +to the third party based on the extent of your activity of conveying +the work, and under which the third party grants, to any of the +parties who would receive the covered work from you, a discriminatory +patent license (a) in connection with copies of the covered work +conveyed by you (or copies made from those copies), or (b) primarily +for and in connection with specific products or compilations that +contain the covered work, unless you entered into that arrangement, +or that patent license was granted, prior to 28 March 2007. + + Nothing in this License shall be construed as excluding or limiting +any implied license or other defenses to infringement that may +otherwise be available to you under applicable patent law. + + 12. No Surrender of Others' Freedom. + + If conditions are imposed on you (whether by court order, agreement or +otherwise) that contradict the conditions of this License, they do not +excuse you from the conditions of this License. If you cannot convey a +covered work so as to satisfy simultaneously your obligations under this +License and any other pertinent obligations, then as a consequence you may +not convey it at all. For example, if you agree to terms that obligate you +to collect a royalty for further conveying from those to whom you convey +the Program, the only way you could satisfy both those terms and this +License would be to refrain entirely from conveying the Program. + + 13. Remote Network Interaction; Use with the GNU General Public License. + + Notwithstanding any other provision of this License, if you modify the +Program, your modified version must prominently offer all users +interacting with it remotely through a computer network (if your version +supports such interaction) an opportunity to receive the Corresponding +Source of your version by providing access to the Corresponding Source +from a network server at no charge, through some standard or customary +means of facilitating copying of software. This Corresponding Source +shall include the Corresponding Source for any work covered by version 3 +of the GNU General Public License that is incorporated pursuant to the +following paragraph. + + Notwithstanding any other provision of this License, you have +permission to link or combine any covered work with a work licensed +under version 3 of the GNU General Public License into a single +combined work, and to convey the resulting work. The terms of this +License will continue to apply to the part which is the covered work, +but the work with which it is combined will remain governed by version +3 of the GNU General Public License. + + 14. Revised Versions of this License. + + The Free Software Foundation may publish revised and/or new versions of +the GNU Affero General Public License from time to time. Such new versions +will be similar in spirit to the present version, but may differ in detail to +address new problems or concerns. + + Each version is given a distinguishing version number. If the +Program specifies that a certain numbered version of the GNU Affero General +Public License "or any later version" applies to it, you have the +option of following the terms and conditions either of that numbered +version or of any later version published by the Free Software +Foundation. If the Program does not specify a version number of the +GNU Affero General Public License, you may choose any version ever published +by the Free Software Foundation. + + If the Program specifies that a proxy can decide which future +versions of the GNU Affero General Public License can be used, that proxy's +public statement of acceptance of a version permanently authorizes you +to choose that version for the Program. + + Later license versions may give you additional or different +permissions. However, no additional obligations are imposed on any +author or copyright holder as a result of your choosing to follow a +later version. + + 15. Disclaimer of Warranty. + + THERE IS NO WARRANTY FOR THE PROGRAM, TO THE EXTENT PERMITTED BY +APPLICABLE LAW. EXCEPT WHEN OTHERWISE STATED IN WRITING THE COPYRIGHT +HOLDERS AND/OR OTHER PARTIES PROVIDE THE PROGRAM "AS IS" WITHOUT WARRANTY +OF ANY KIND, EITHER EXPRESSED OR IMPLIED, INCLUDING, BUT NOT LIMITED TO, +THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR +PURPOSE. THE ENTIRE RISK AS TO THE QUALITY AND PERFORMANCE OF THE PROGRAM +IS WITH YOU. SHOULD THE PROGRAM PROVE DEFECTIVE, YOU ASSUME THE COST OF +ALL NECESSARY SERVICING, REPAIR OR CORRECTION. + + 16. Limitation of Liability. + + IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN WRITING +WILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MODIFIES AND/OR CONVEYS +THE PROGRAM AS PERMITTED ABOVE, BE LIABLE TO YOU FOR DAMAGES, INCLUDING ANY +GENERAL, SPECIAL, INCIDENTAL OR CONSEQUENTIAL DAMAGES ARISING OUT OF THE +USE OR INABILITY TO USE THE PROGRAM (INCLUDING BUT NOT LIMITED TO LOSS OF +DATA OR DATA BEING RENDERED INACCURATE OR LOSSES SUSTAINED BY YOU OR THIRD +PARTIES OR A FAILURE OF THE PROGRAM TO OPERATE WITH ANY OTHER PROGRAMS), +EVEN IF SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE POSSIBILITY OF +SUCH DAMAGES. + + 17. Interpretation of Sections 15 and 16. + + If the disclaimer of warranty and limitation of liability provided +above cannot be given local legal effect according to their terms, +reviewing courts shall apply local law that most closely approximates +an absolute waiver of all civil liability in connection with the +Program, unless a warranty or assumption of liability accompanies a +copy of the Program in return for a fee. + + END OF TERMS AND CONDITIONS + + How to Apply These Terms to Your New Programs + + If you develop a new program, and you want it to be of the greatest +possible use to the public, the best way to achieve this is to make it +free software which everyone can redistribute and change under these terms. + + To do so, attach the following notices to the program. It is safest +to attach them to the start of each source file to most effectively +state the exclusion of warranty; and each file should have at least +the "copyright" line and a pointer to where the full notice is found. + + + Copyright (C) + + This program is free software: you can redistribute it and/or modify + it under the terms of the GNU Affero General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU Affero General Public License for more details. + + You should have received a copy of the GNU Affero General Public License + along with this program. If not, see . + +Also add information on how to contact you by electronic and paper mail. + + If your software can interact with users remotely through a computer +network, you should also make sure that it provides a way for users to +get its source. For example, if your program is a web application, its +interface could display a "Source" link that leads users to an archive +of the code. There are many ways you could offer source, and different +solutions will be better for different programs; see section 13 for the +specific requirements. + + You should also get your employer (if you work as a programmer) or school, +if any, to sign a "copyright disclaimer" for the program, if necessary. +For more information on this, and how to apply and follow the GNU AGPL, see +. diff --git a/services/real-time/package.json b/services/real-time/package.json index a806d93ab6..3a2145a85e 100644 --- a/services/real-time/package.json +++ b/services/real-time/package.json @@ -3,6 +3,8 @@ "version": "0.1.4", "description": "The socket.io layer of ShareLaTeX for real-time editor interactions", "author": "ShareLaTeX ", + "license": "AGPL-3.0-only", + "private": true, "repository": { "type": "git", "url": "https://github.com/sharelatex/real-time-sharelatex.git" From c2f8f26da417bde411f43a295a03959079ca3bce Mon Sep 17 00:00:00 2001 From: Jakob Ackermann Date: Tue, 5 Jan 2021 18:26:52 +0000 Subject: [PATCH 457/491] [misc] bump the node version to 10.23.1 --- services/real-time/.nvmrc | 2 +- services/real-time/Dockerfile | 2 +- services/real-time/buildscript.txt | 2 +- services/real-time/docker-compose.yml | 4 ++-- services/real-time/package-lock.json | 8 ++++---- 5 files changed, 9 insertions(+), 9 deletions(-) diff --git a/services/real-time/.nvmrc b/services/real-time/.nvmrc index c2f6421352..2baa2d433a 100644 --- a/services/real-time/.nvmrc +++ b/services/real-time/.nvmrc @@ -1 +1 @@ -10.22.1 +10.23.1 diff --git a/services/real-time/Dockerfile b/services/real-time/Dockerfile index f0e362fca0..2da67d2436 100644 --- a/services/real-time/Dockerfile +++ b/services/real-time/Dockerfile @@ -2,7 +2,7 @@ # Instead run bin/update_build_scripts from # https://github.com/sharelatex/sharelatex-dev-environment -FROM node:10.22.1 as base +FROM node:10.23.1 as base WORKDIR /app diff --git a/services/real-time/buildscript.txt b/services/real-time/buildscript.txt index f75047baec..6fbccbe4c4 100644 --- a/services/real-time/buildscript.txt +++ b/services/real-time/buildscript.txt @@ -3,6 +3,6 @@ real-time --docker-repos=gcr.io/overleaf-ops --env-add= --env-pass-through= ---node-version=10.22.1 +--node-version=10.23.1 --public-repo=True --script-version=3.4.0 diff --git a/services/real-time/docker-compose.yml b/services/real-time/docker-compose.yml index eac74fbee2..8233095495 100644 --- a/services/real-time/docker-compose.yml +++ b/services/real-time/docker-compose.yml @@ -6,7 +6,7 @@ version: "2.3" services: test_unit: - image: node:10.22.1 + image: node:10.23.1 volumes: - .:/app working_dir: /app @@ -18,7 +18,7 @@ services: user: node test_acceptance: - image: node:10.22.1 + image: node:10.23.1 volumes: - .:/app working_dir: /app diff --git a/services/real-time/package-lock.json b/services/real-time/package-lock.json index 0f383213a0..ac3e97ade9 100644 --- a/services/real-time/package-lock.json +++ b/services/real-time/package-lock.json @@ -1241,7 +1241,7 @@ "bintrees": { "version": "1.0.1", "resolved": "https://registry.npmjs.org/bintrees/-/bintrees-1.0.1.tgz", - "integrity": "sha512-tbaUB1QpTIj4cKY8c1rvNAvEQXA+ekzHmbe4jzNfW3QWsF9GnnP/BRWyl6/qqS53heoYJ93naaFcm/jooONH8g==" + "integrity": "sha1-DmVcm5wkNeqraL9AJyJtK1WjRSQ=" }, "body-parser": { "version": "1.19.0", @@ -2445,7 +2445,7 @@ "findit2": { "version": "2.2.3", "resolved": "https://registry.npmjs.org/findit2/-/findit2-2.2.3.tgz", - "integrity": "sha512-lg/Moejf4qXovVutL0Lz4IsaPoNYMuxt4PA0nGqFxnJ1CTTGGlEO2wKgoDpwknhvZ8k4Q2F+eesgkLbG2Mxfog==" + "integrity": "sha1-WKRmaX34piBc39vzlVNri9d3pfY=" }, "flat-cache": { "version": "2.0.1", @@ -3727,7 +3727,7 @@ "module-details-from-path": { "version": "1.0.3", "resolved": "https://registry.npmjs.org/module-details-from-path/-/module-details-from-path-1.0.3.tgz", - "integrity": "sha512-ySViT69/76t8VhE1xXHK6Ch4NcDd26gx0MzKXLO+F7NOtnqH68d9zF94nT8ZWSxXh8ELOERsnJO/sWt1xZYw5A==" + "integrity": "sha1-EUyUlnPiqKNenTV4hSeqN7Z52is=" }, "moment": { "version": "2.27.0", @@ -5781,7 +5781,7 @@ "tdigest": { "version": "0.1.1", "resolved": "https://registry.npmjs.org/tdigest/-/tdigest-0.1.1.tgz", - "integrity": "sha512-CXcDY/NIgIbKZPx5H4JJNpq6JwJhU5Z4+yWj4ZghDc7/9nVajiRlPPyMXRePPPlBfcayUqtoCXjo7/Hm82ecUA==", + "integrity": "sha1-Ljyyw56kSeVdHmzZEReszKRYgCE=", "requires": { "bintrees": "1.0.1" } From c308e60b6bdf3c84c2c8bbfa8c34f65d0793c0a7 Mon Sep 17 00:00:00 2001 From: Henry Oswald Date: Mon, 8 Feb 2021 11:13:22 +0000 Subject: [PATCH 458/491] speed up DocumentUpdaterControllerTests by directly injecting redis into tests --- .../unit/js/DocumentUpdaterControllerTests.js | 41 ++++++++++--------- 1 file changed, 22 insertions(+), 19 deletions(-) diff --git a/services/real-time/test/unit/js/DocumentUpdaterControllerTests.js b/services/real-time/test/unit/js/DocumentUpdaterControllerTests.js index 832ad9490a..738375b32a 100644 --- a/services/real-time/test/unit/js/DocumentUpdaterControllerTests.js +++ b/services/real-time/test/unit/js/DocumentUpdaterControllerTests.js @@ -20,14 +20,14 @@ const modulePath = require('path').join( const MockClient = require('./helpers/MockClient') describe('DocumentUpdaterController', function () { - beforeEach(function () { + beforeEach(function (done) { this.project_id = 'project-id-123' this.doc_id = 'doc-id-123' this.callback = sinon.stub() this.io = { mock: 'socket.io' } this.rclient = [] this.RoomEvents = { on: sinon.stub() } - return (this.EditorUpdatesController = SandboxedModule.require(modulePath, { + this.EditorUpdatesController = SandboxedModule.require(modulePath, { requires: { 'logger-sharelatex': (this.logger = { error: sinon.stub(), @@ -46,13 +46,17 @@ describe('DocumentUpdaterController', function () { pubsub: null } }), - '@overleaf/redis-wrapper': (this.redis = { - createClient: (name) => { - let rclientStub - this.rclient.push((rclientStub = { name })) - return rclientStub + './RedisClientManager': { + createClientList: () => { + this.redis = { + createClient: (name) => { + let rclientStub + this.rclient.push((rclientStub = { name })) + return rclientStub + } + } } - }), + }, './SafeJsonParse': (this.SafeJsonParse = { parse: (data, cb) => cb(null, JSON.parse(data)) }), @@ -64,7 +68,8 @@ describe('DocumentUpdaterController', function () { }), './ChannelManager': (this.ChannelManager = {}) } - })) + }) + done() }) describe('listenForUpdatesFromDocumentUpdater', function () { @@ -78,22 +83,20 @@ describe('DocumentUpdaterController', function () { this.rclient[0].on = sinon.stub() this.rclient[1].subscribe = sinon.stub() this.rclient[1].on = sinon.stub() - return this.EditorUpdatesController.listenForUpdatesFromDocumentUpdater() + this.EditorUpdatesController.listenForUpdatesFromDocumentUpdater() }) it('should subscribe to the doc-updater stream', function () { - return this.rclient[0].subscribe - .calledWith('applied-ops') - .should.equal(true) + this.rclient[0].subscribe.calledWith('applied-ops').should.equal(true) }) it('should register a callback to handle updates', function () { - return this.rclient[0].on.calledWith('message').should.equal(true) + this.rclient[0].on.calledWith('message').should.equal(true) }) - return it('should subscribe to any additional doc-updater stream', function () { + it('should subscribe to any additional doc-updater stream', function () { this.rclient[1].subscribe.calledWith('applied-ops').should.equal(true) - return this.rclient[1].on.calledWith('message').should.equal(true) + this.rclient[1].on.calledWith('message').should.equal(true) }) }) @@ -110,7 +113,7 @@ describe('DocumentUpdaterController', function () { ) }) - return it('should log an error', function () { + it('should log an error', function () { return this.logger.error.called.should.equal(true) }) }) @@ -129,14 +132,14 @@ describe('DocumentUpdaterController', function () { ) }) - return it('should apply the update', function () { + it('should apply the update', function () { return this.EditorUpdatesController._applyUpdateFromDocumentUpdater .calledWith(this.io, this.doc_id, this.message.op) .should.equal(true) }) }) - return describe('with error', function () { + describe('with error', function () { beforeEach(function () { this.message = { doc_id: this.doc_id, From 31e1808dd89b717da096f20051eb2ed90e56e0f5 Mon Sep 17 00:00:00 2001 From: Henry Oswald Date: Mon, 8 Feb 2021 12:19:25 +0000 Subject: [PATCH 459/491] shard the pending-updates-list queue --- .../app/js/DocumentUpdaterManager.js | 25 ++++++++-- .../real-time/config/settings.defaults.js | 6 +++ .../unit/js/DocumentUpdaterControllerTests.js | 3 +- .../unit/js/DocumentUpdaterManagerTests.js | 47 +++++++++++++++---- 4 files changed, 65 insertions(+), 16 deletions(-) diff --git a/services/real-time/app/js/DocumentUpdaterManager.js b/services/real-time/app/js/DocumentUpdaterManager.js index 925d9f38d8..aacadd9391 100644 --- a/services/real-time/app/js/DocumentUpdaterManager.js +++ b/services/real-time/app/js/DocumentUpdaterManager.js @@ -88,6 +88,15 @@ const DocumentUpdaterManager = { }) }, + _getPendingUpdateListKey() { + const shard = _.random(0, settings.pendingUpdateListShardCount) + if (shard === 0) { + return 'pending-updates-list' + } else { + return `pending-updates-list-${shard}` + } + }, + queueChange(project_id, doc_id, change, callback) { const allowedKeys = [ 'doc', @@ -123,12 +132,18 @@ const DocumentUpdaterManager = { error = new OError('error pushing update into redis').withCause(error) return callback(error) } - rclient.rpush('pending-updates-list', doc_key, function (error) { - if (error) { - error = new OError('error pushing doc_id into redis').withCause(error) + rclient.rpush( + DocumentUpdaterManager._getPendingUpdateListKey(), + doc_key, + function (error) { + if (error) { + error = new OError('error pushing doc_id into redis').withCause( + error + ) + } + callback(error) } - callback(error) - }) + ) }) } } diff --git a/services/real-time/config/settings.defaults.js b/services/real-time/config/settings.defaults.js index 5c0d501b50..1d7f0d4c93 100644 --- a/services/real-time/config/settings.defaults.js +++ b/services/real-time/config/settings.defaults.js @@ -114,6 +114,12 @@ const settings = { max_doc_length: 2 * 1024 * 1024, // 2mb + // should be set to the same same as dispatcherCount in document updater + pendingUpdateListShardCount: parseInt( + process.env.PENDING_UPDATE_LIST_SHARD_COUNT || 1, + 10 + ), + // combine // max_doc_length (2mb see above) * 2 (delete + insert) // max_ranges_size (3mb see MAX_RANGES_SIZE in document-updater) diff --git a/services/real-time/test/unit/js/DocumentUpdaterControllerTests.js b/services/real-time/test/unit/js/DocumentUpdaterControllerTests.js index 738375b32a..0aabc51de7 100644 --- a/services/real-time/test/unit/js/DocumentUpdaterControllerTests.js +++ b/services/real-time/test/unit/js/DocumentUpdaterControllerTests.js @@ -20,7 +20,7 @@ const modulePath = require('path').join( const MockClient = require('./helpers/MockClient') describe('DocumentUpdaterController', function () { - beforeEach(function (done) { + beforeEach(function () { this.project_id = 'project-id-123' this.doc_id = 'doc-id-123' this.callback = sinon.stub() @@ -69,7 +69,6 @@ describe('DocumentUpdaterController', function () { './ChannelManager': (this.ChannelManager = {}) } }) - done() }) describe('listenForUpdatesFromDocumentUpdater', function () { diff --git a/services/real-time/test/unit/js/DocumentUpdaterManagerTests.js b/services/real-time/test/unit/js/DocumentUpdaterManagerTests.js index d6da97ddab..e3c108b824 100644 --- a/services/real-time/test/unit/js/DocumentUpdaterManagerTests.js +++ b/services/real-time/test/unit/js/DocumentUpdaterManagerTests.js @@ -15,6 +15,7 @@ const sinon = require('sinon') const SandboxedModule = require('sandboxed-module') const path = require('path') const modulePath = '../../../app/js/DocumentUpdaterManager' +const _ = require('underscore') describe('DocumentUpdaterManager', function () { beforeEach(function () { @@ -34,7 +35,8 @@ describe('DocumentUpdaterManager', function () { } } }, - maxUpdateSize: 7 * 1024 * 1024 + maxUpdateSize: 7 * 1024 * 1024, + pendingUpdateListShardCount: 10 } this.rclient = { auth() {} } @@ -256,7 +258,7 @@ describe('DocumentUpdaterManager', function () { }) }) - return describe('queueChange', function () { + describe('queueChange', function () { beforeEach(function () { this.change = { doc: '1234567890', @@ -269,7 +271,12 @@ describe('DocumentUpdaterManager', function () { describe('successfully', function () { beforeEach(function () { - return this.DocumentUpdaterManager.queueChange( + this.pendingUpdateListKey = `pending-updates-list-key-${Math.random()}` + + this.DocumentUpdaterManager._getPendingUpdateListKey = sinon + .stub() + .returns(this.pendingUpdateListKey) + this.DocumentUpdaterManager.queueChange( this.project_id, this.doc_id, this.change, @@ -278,7 +285,7 @@ describe('DocumentUpdaterManager', function () { }) it('should push the change', function () { - return this.rclient.rpush + this.rclient.rpush .calledWith( `PendingUpdates:${this.doc_id}`, JSON.stringify(this.change) @@ -286,10 +293,10 @@ describe('DocumentUpdaterManager', function () { .should.equal(true) }) - return it('should notify the doc updater of the change via the pending-updates-list queue', function () { - return this.rclient.rpush + it('should notify the doc updater of the change via the pending-updates-list queue', function () { + this.rclient.rpush .calledWith( - 'pending-updates-list', + this.pendingUpdateListKey, `${this.project_id}:${this.doc_id}` ) .should.equal(true) @@ -366,7 +373,7 @@ describe('DocumentUpdaterManager', function () { }) }) - return describe('with invalid keys', function () { + describe('with invalid keys', function () { beforeEach(function () { this.change = { op: [{ d: 'test', p: 345 }], @@ -380,7 +387,7 @@ describe('DocumentUpdaterManager', function () { ) }) - return it('should remove the invalid keys from the change', function () { + it('should remove the invalid keys from the change', function () { return this.rclient.rpush .calledWith( `PendingUpdates:${this.doc_id}`, @@ -390,4 +397,26 @@ describe('DocumentUpdaterManager', function () { }) }) }) + + describe('_getPendingUpdateListKey', function () { + beforeEach(function () { + const keys = _.times( + 10000, + this.DocumentUpdaterManager._getPendingUpdateListKey + ) + this.keys = _.unique(keys) + }) + it('should return normal pending updates key', function () { + const key = this.DocumentUpdaterManager._getPendingUpdateListKey() + _.contains(this.keys, 'pending-updates-list').should.equal(true) + }) + + it('should return pending-updates-list-n keys', function () { + _.contains(this.keys, 'pending-updates-list-1').should.equal(true) + _.contains(this.keys, 'pending-updates-list-3').should.equal(true) + }) + it('should not include pending-updates-list-0 key', function () { + _.contains(this.keys, 'pending-updates-list-0').should.equal(false) + }) + }) }) From 46b389e8b35960a74ab41e62701f675a3226f23a Mon Sep 17 00:00:00 2001 From: Henry Oswald Date: Tue, 9 Feb 2021 10:48:40 +0000 Subject: [PATCH 460/491] fix off by 1 error in key sharding --- services/real-time/app/js/DocumentUpdaterManager.js | 2 +- .../real-time/test/unit/js/DocumentUpdaterManagerTests.js | 7 ++++++- 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/services/real-time/app/js/DocumentUpdaterManager.js b/services/real-time/app/js/DocumentUpdaterManager.js index aacadd9391..0aa0d23466 100644 --- a/services/real-time/app/js/DocumentUpdaterManager.js +++ b/services/real-time/app/js/DocumentUpdaterManager.js @@ -89,7 +89,7 @@ const DocumentUpdaterManager = { }, _getPendingUpdateListKey() { - const shard = _.random(0, settings.pendingUpdateListShardCount) + const shard = _.random(0, settings.pendingUpdateListShardCount - 1) if (shard === 0) { return 'pending-updates-list' } else { diff --git a/services/real-time/test/unit/js/DocumentUpdaterManagerTests.js b/services/real-time/test/unit/js/DocumentUpdaterManagerTests.js index e3c108b824..2a5fdf530f 100644 --- a/services/real-time/test/unit/js/DocumentUpdaterManagerTests.js +++ b/services/real-time/test/unit/js/DocumentUpdaterManagerTests.js @@ -407,16 +407,21 @@ describe('DocumentUpdaterManager', function () { this.keys = _.unique(keys) }) it('should return normal pending updates key', function () { - const key = this.DocumentUpdaterManager._getPendingUpdateListKey() _.contains(this.keys, 'pending-updates-list').should.equal(true) }) it('should return pending-updates-list-n keys', function () { _.contains(this.keys, 'pending-updates-list-1').should.equal(true) _.contains(this.keys, 'pending-updates-list-3').should.equal(true) + _.contains(this.keys, 'pending-updates-list-9').should.equal(true) }) + it('should not include pending-updates-list-0 key', function () { _.contains(this.keys, 'pending-updates-list-0').should.equal(false) }) + + it('should not include maximum as pendingUpdateListShardCount value', function () { + _.contains(this.keys, 'pending-updates-list-10').should.equal(false) + }) }) }) From 49bed6595d27e711c3521708597373bf6900bfea Mon Sep 17 00:00:00 2001 From: Henry Oswald Date: Tue, 9 Feb 2021 12:42:58 +0000 Subject: [PATCH 461/491] added queueKey to error info when trying to push to redis --- .../app/js/DocumentUpdaterManager.js | 19 ++++++++----------- 1 file changed, 8 insertions(+), 11 deletions(-) diff --git a/services/real-time/app/js/DocumentUpdaterManager.js b/services/real-time/app/js/DocumentUpdaterManager.js index 0aa0d23466..39bef96ebd 100644 --- a/services/real-time/app/js/DocumentUpdaterManager.js +++ b/services/real-time/app/js/DocumentUpdaterManager.js @@ -132,18 +132,15 @@ const DocumentUpdaterManager = { error = new OError('error pushing update into redis').withCause(error) return callback(error) } - rclient.rpush( - DocumentUpdaterManager._getPendingUpdateListKey(), - doc_key, - function (error) { - if (error) { - error = new OError('error pushing doc_id into redis').withCause( - error - ) - } - callback(error) + const queueKey = DocumentUpdaterManager._getPendingUpdateListKey() + rclient.rpush(queueKey, doc_key, function (error) { + if (error) { + error = new OError('error pushing doc_id into redis') + .withInfo({ queueKey }) + .withCause(error) } - ) + callback(error) + }) }) } } From 27e7b77bdb2b0992b160ccf4d2e965755a3101e3 Mon Sep 17 00:00:00 2001 From: Henry Oswald Date: Tue, 16 Feb 2021 10:53:35 +0000 Subject: [PATCH 462/491] bump default key shards to 10 --- services/real-time/config/settings.defaults.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/services/real-time/config/settings.defaults.js b/services/real-time/config/settings.defaults.js index 1d7f0d4c93..486b686083 100644 --- a/services/real-time/config/settings.defaults.js +++ b/services/real-time/config/settings.defaults.js @@ -116,7 +116,7 @@ const settings = { // should be set to the same same as dispatcherCount in document updater pendingUpdateListShardCount: parseInt( - process.env.PENDING_UPDATE_LIST_SHARD_COUNT || 1, + process.env.PENDING_UPDATE_LIST_SHARD_COUNT || 10, 10 ), From b7b13a90d1e4ad7f9747838987984cb1eaf868f1 Mon Sep 17 00:00:00 2001 From: Jakob Ackermann Date: Fri, 26 Feb 2021 00:58:50 +0100 Subject: [PATCH 463/491] [misc] ApplyUpdateTests: process all the pending-updates-lists --- .../test/acceptance/js/ApplyUpdateTests.js | 40 +++++++++++++++++-- 1 file changed, 36 insertions(+), 4 deletions(-) diff --git a/services/real-time/test/acceptance/js/ApplyUpdateTests.js b/services/real-time/test/acceptance/js/ApplyUpdateTests.js index 8a750fd68d..27f77b973f 100644 --- a/services/real-time/test/acceptance/js/ApplyUpdateTests.js +++ b/services/real-time/test/acceptance/js/ApplyUpdateTests.js @@ -26,6 +26,38 @@ const rclient = redis.createClient(settings.redis.documentupdater) const redisSettings = settings.redis +const PENDING_UPDATES_LIST_KEYS = [0, 1, 2, 3, 4, 5, 6, 7, 8, 9].map((n) => { + let key = 'pending-updates-list' + if (n !== 0) { + key += `-${n}` + } + return key +}) + +function getPendingUpdatesList(cb) { + Promise.all( + PENDING_UPDATES_LIST_KEYS.map((key) => rclient.lrange(key, 0, -1)) + ) + .then((results) => { + cb( + null, + results.reduce((acc, more) => { + if (more.length) { + acc.push(...more) + } + return acc + }, []) + ) + }) + .catch(cb) +} + +function clearPendingUpdatesList(cb) { + Promise.all(PENDING_UPDATES_LIST_KEYS.map((key) => rclient.del(key))) + .then(() => cb(null)) + .catch(cb) +} + describe('applyOtUpdate', function () { before(function () { return (this.update = { @@ -91,7 +123,7 @@ describe('applyOtUpdate', function () { }) it('should push the doc into the pending updates list', function (done) { - rclient.lrange('pending-updates-list', 0, -1, (error, ...rest) => { + getPendingUpdatesList((error, ...rest) => { const [doc_id] = Array.from(rest[0]) doc_id.should.equal(`${this.project_id}:${this.doc_id}`) return done() @@ -123,7 +155,7 @@ describe('applyOtUpdate', function () { return after(function (done) { return async.series( [ - (cb) => rclient.del('pending-updates-list', cb), + (cb) => clearPendingUpdatesList(cb), (cb) => rclient.del( 'DocsWithPendingUpdates', @@ -393,7 +425,7 @@ describe('applyOtUpdate', function () { }) it('should push the doc into the pending updates list', function (done) { - rclient.lrange('pending-updates-list', 0, -1, (error, ...rest) => { + getPendingUpdatesList((error, ...rest) => { const [doc_id] = Array.from(rest[0]) doc_id.should.equal(`${this.project_id}:${this.doc_id}`) return done() @@ -425,7 +457,7 @@ describe('applyOtUpdate', function () { return after(function (done) { return async.series( [ - (cb) => rclient.del('pending-updates-list', cb), + (cb) => clearPendingUpdatesList(cb), (cb) => rclient.del( 'DocsWithPendingUpdates', From a26a497743563ccd751b7f21bd6449fd6fb5c0c8 Mon Sep 17 00:00:00 2001 From: Jakob Ackermann Date: Mon, 1 Mar 2021 15:48:27 +0000 Subject: [PATCH 464/491] [misc] let mocha catch errors when running acceptance tests --- services/real-time/config/settings.test.js | 5 +++++ 1 file changed, 5 insertions(+) create mode 100644 services/real-time/config/settings.test.js diff --git a/services/real-time/config/settings.test.js b/services/real-time/config/settings.test.js new file mode 100644 index 0000000000..7e2e0b24b6 --- /dev/null +++ b/services/real-time/config/settings.test.js @@ -0,0 +1,5 @@ +module.exports = { + errors: { + catchUncaughtErrors: false + } +} From b9e7cbf73b39ee9ca3652821e5309d7e78adbaab Mon Sep 17 00:00:00 2001 From: Eric Mc Sween Date: Thu, 18 Mar 2021 16:19:31 -0400 Subject: [PATCH 465/491] Add global test setup Configure chai and SandboxedModule in a central place. Configure SandboxedModule globals that are needed in Node 12. --- services/real-time/.mocharc.json | 3 + services/real-time/package-lock.json | 902 ++++++++++++++---- services/real-time/package.json | 4 +- .../test/acceptance/js/ApplyUpdateTests.js | 4 +- .../test/acceptance/js/ClientTrackingTests.js | 4 +- .../test/acceptance/js/JoinDocTests.js | 4 +- .../test/acceptance/js/JoinProjectTests.js | 4 +- .../test/acceptance/js/LeaveDocTests.js | 4 +- .../test/acceptance/js/MatrixTests.js | 3 +- .../test/acceptance/js/ReceiveUpdateTests.js | 4 +- .../test/acceptance/js/SessionTests.js | 3 +- services/real-time/test/setup.js | 38 + .../test/unit/js/AuthorizationManagerTests.js | 4 +- .../test/unit/js/ChannelManagerTests.js | 9 +- .../unit/js/ConnectedUsersManagerTests.js | 2 - .../unit/js/DocumentUpdaterControllerTests.js | 6 - .../unit/js/DocumentUpdaterManagerTests.js | 17 +- .../test/unit/js/DrainManagerTests.js | 7 +- .../test/unit/js/EventLoggerTests.js | 5 - .../test/unit/js/RoomManagerTests.js | 9 +- .../test/unit/js/SafeJsonParseTest.js | 5 +- .../test/unit/js/WebApiManagerTests.js | 6 - .../test/unit/js/WebsocketControllerTests.js | 9 +- .../unit/js/WebsocketLoadBalancerTests.js | 5 - 24 files changed, 757 insertions(+), 304 deletions(-) create mode 100644 services/real-time/.mocharc.json create mode 100644 services/real-time/test/setup.js diff --git a/services/real-time/.mocharc.json b/services/real-time/.mocharc.json new file mode 100644 index 0000000000..dc3280aa96 --- /dev/null +++ b/services/real-time/.mocharc.json @@ -0,0 +1,3 @@ +{ + "require": "test/setup.js" +} diff --git a/services/real-time/package-lock.json b/services/real-time/package-lock.json index ac3e97ade9..4ba3fdc753 100644 --- a/services/real-time/package-lock.json +++ b/services/real-time/package-lock.json @@ -826,7 +826,7 @@ "@protobufjs/aspromise": { "version": "1.1.2", "resolved": "https://registry.npmjs.org/@protobufjs/aspromise/-/aspromise-1.1.2.tgz", - "integrity": "sha512-j+gKExEuLmKwvz3OgROXtrJ2UG2x8Ch2YZUxahh+s1F2HZ+wAceUNLkvy6zKCPVRkU++ZWQrdxsUeQXmcg4uoQ==" + "integrity": "sha1-m4sMxmPWaafY9vXQiToU00jzD78=" }, "@protobufjs/base64": { "version": "1.1.2", @@ -841,12 +841,12 @@ "@protobufjs/eventemitter": { "version": "1.1.0", "resolved": "https://registry.npmjs.org/@protobufjs/eventemitter/-/eventemitter-1.1.0.tgz", - "integrity": "sha512-j9ednRT81vYJ9OfVuXG6ERSTdEL1xVsNgqpkxMsbIabzSo3goCjDIveeGv5d03om39ML71RdmrGNjG5SReBP/Q==" + "integrity": "sha1-NVy8mLr61ZePntCV85diHx0Ga3A=" }, "@protobufjs/fetch": { "version": "1.1.0", "resolved": "https://registry.npmjs.org/@protobufjs/fetch/-/fetch-1.1.0.tgz", - "integrity": "sha512-lljVXpqXebpsijW71PZaCYeIcE5on1w5DlQy5WH6GLbFryLUrBD4932W/E2BSpfRJWseIL4v/KPgBFxDOIdKpQ==", + "integrity": "sha1-upn7WYYUr2VwDBYZ/wbUVLDYTEU=", "requires": { "@protobufjs/aspromise": "^1.1.1", "@protobufjs/inquire": "^1.1.0" @@ -855,27 +855,78 @@ "@protobufjs/float": { "version": "1.0.2", "resolved": "https://registry.npmjs.org/@protobufjs/float/-/float-1.0.2.tgz", - "integrity": "sha512-Ddb+kVXlXst9d+R9PfTIxh1EdNkgoRe5tOX6t01f1lYWOvJnSPDBlG241QLzcyPdoNTsblLUdujGSE4RzrTZGQ==" + "integrity": "sha1-Xp4avctz/Ap8uLKR33jIy9l7h9E=" }, "@protobufjs/inquire": { "version": "1.1.0", "resolved": "https://registry.npmjs.org/@protobufjs/inquire/-/inquire-1.1.0.tgz", - "integrity": "sha512-kdSefcPdruJiFMVSbn801t4vFK7KB/5gd2fYvrxhuJYg8ILrmn9SKSX2tZdV6V+ksulWqS7aXjBcRXl3wHoD9Q==" + "integrity": "sha1-/yAOPnzyQp4tyvwRQIKOjMY48Ik=" }, "@protobufjs/path": { "version": "1.1.2", "resolved": "https://registry.npmjs.org/@protobufjs/path/-/path-1.1.2.tgz", - "integrity": "sha512-6JOcJ5Tm08dOHAbdR3GrvP+yUUfkjG5ePsHYczMFLq3ZmMkAD98cDgcT2iA1lJ9NVwFd4tH/iSSoe44YWkltEA==" + "integrity": "sha1-bMKyDFya1q0NzP0hynZz2Nf79o0=" }, "@protobufjs/pool": { "version": "1.1.0", "resolved": "https://registry.npmjs.org/@protobufjs/pool/-/pool-1.1.0.tgz", - "integrity": "sha512-0kELaGSIDBKvcgS4zkjz1PeddatrjYcmMWOlAuAPwAeccUrPHdUqo/J6LiymHHEiJT5NrF1UVwxY14f+fy4WQw==" + "integrity": "sha1-Cf0V8tbTq/qbZbw2ZQbWrXhG/1Q=" }, "@protobufjs/utf8": { "version": "1.1.0", "resolved": "https://registry.npmjs.org/@protobufjs/utf8/-/utf8-1.1.0.tgz", - "integrity": "sha512-Vvn3zZrhQZkkBE8LSuW3em98c0FwgO4nxzv6OdSxPKJIEKY2bGbHn+mhGIPerzI4twdxaP8/0+06HBpwf345Lw==" + "integrity": "sha1-p3c2C1s5oaLlEG+OhY8v0tBgxXA=" + }, + "@sinonjs/commons": { + "version": "1.8.2", + "resolved": "https://registry.npmjs.org/@sinonjs/commons/-/commons-1.8.2.tgz", + "integrity": "sha512-sruwd86RJHdsVf/AtBoijDmUqJp3B6hF/DGC23C+JaegnDHaZyewCjoVGTdg3J0uz3Zs7NnIT05OBOmML72lQw==", + "dev": true, + "requires": { + "type-detect": "4.0.8" + }, + "dependencies": { + "type-detect": { + "version": "4.0.8", + "resolved": "https://registry.npmjs.org/type-detect/-/type-detect-4.0.8.tgz", + "integrity": "sha512-0fr/mIH1dlO+x7TlcMy+bIDqKPsw/70tVyeHW787goQjhmqaZe10uwLujubK9q9Lg6Fiho1KUKDYz0Z7k7g5/g==", + "dev": true + } + } + }, + "@sinonjs/fake-timers": { + "version": "6.0.1", + "resolved": "https://registry.npmjs.org/@sinonjs/fake-timers/-/fake-timers-6.0.1.tgz", + "integrity": "sha512-MZPUxrmFubI36XS1DI3qmI0YdN1gks62JtFZvxR67ljjSNCeK6U08Zx4msEWOXuofgqUt6zPHSi1H9fbjR/NRA==", + "dev": true, + "requires": { + "@sinonjs/commons": "^1.7.0" + } + }, + "@sinonjs/samsam": { + "version": "5.3.1", + "resolved": "https://registry.npmjs.org/@sinonjs/samsam/-/samsam-5.3.1.tgz", + "integrity": "sha512-1Hc0b1TtyfBu8ixF/tpfSHTVWKwCBLY4QJbkgnE7HcwyvT2xArDxb4K7dMgqRm3szI+LJbzmW/s4xxEhv6hwDg==", + "dev": true, + "requires": { + "@sinonjs/commons": "^1.6.0", + "lodash.get": "^4.4.2", + "type-detect": "^4.0.8" + }, + "dependencies": { + "type-detect": { + "version": "4.0.8", + "resolved": "https://registry.npmjs.org/type-detect/-/type-detect-4.0.8.tgz", + "integrity": "sha512-0fr/mIH1dlO+x7TlcMy+bIDqKPsw/70tVyeHW787goQjhmqaZe10uwLujubK9q9Lg6Fiho1KUKDYz0Z7k7g5/g==", + "dev": true + } + } + }, + "@sinonjs/text-encoding": { + "version": "0.7.1", + "resolved": "https://registry.npmjs.org/@sinonjs/text-encoding/-/text-encoding-0.7.1.tgz", + "integrity": "sha512-+iTbntw2IZPb/anVDbypzfQa+ay64MW0Zo8aJ8gZPWMMK6/OubMVb6lUPMagqjOPnmtauXnFCACVl3O7ogjeqQ==", + "dev": true }, "@tootallnate/once": { "version": "1.1.2", @@ -982,6 +1033,12 @@ } } }, + "@ungap/promise-all-settled": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/@ungap/promise-all-settled/-/promise-all-settled-1.1.2.tgz", + "integrity": "sha512-sL/cEvJWAnClXw0wHk85/2L0G6Sj8UB0Ctc1TEMbKSsmpRosqhwj9gWgFRZSrBr2f9tiXISwNhCPmlfqUqyb9Q==", + "dev": true + }, "abbrev": { "version": "1.1.1", "resolved": "https://registry.npmjs.org/abbrev/-/abbrev-1.1.1.tgz", @@ -990,7 +1047,7 @@ "abort-controller": { "version": "3.0.0", "resolved": "https://registry.npmjs.org/abort-controller/-/abort-controller-3.0.0.tgz", - "integrity": "sha512-h8lQ8tacZYnR3vNQTgibj+tODHI5/+l06Au2Pcriv/Gmet0eaj4TwWH41sO9wnHDiQsEj19q0drzdWdeAHtweg==", + "integrity": "sha1-6vVNU7YrrkE46AnKIlyEOabvs5I=", "requires": { "event-target-shim": "^5.0.0" } @@ -1050,6 +1107,12 @@ "uri-js": "^4.2.2" } }, + "ansi-colors": { + "version": "4.1.1", + "resolved": "https://registry.npmjs.org/ansi-colors/-/ansi-colors-4.1.1.tgz", + "integrity": "sha512-JoX0apGbHaUJBNl6yF+p6JAFYZ666/hhCGKN5t9QFjbJQKUU/g8MNbFDbvfrgKXvI1QpZplPOnwIo99lX/AAmA==", + "dev": true + }, "ansi-escapes": { "version": "4.3.1", "resolved": "https://registry.npmjs.org/ansi-escapes/-/ansi-escapes-4.3.1.tgz", @@ -1082,6 +1145,16 @@ "color-convert": "^1.9.0" } }, + "anymatch": { + "version": "3.1.1", + "resolved": "https://registry.npmjs.org/anymatch/-/anymatch-3.1.1.tgz", + "integrity": "sha512-mM8522psRCqzV+6LhomX5wgp25YVibjh8Wj23I5RPkPppSVSjyKD2A2mBJmWGa+KN7f2D6LNh9jkBCeyLktzjg==", + "dev": true, + "requires": { + "normalize-path": "^3.0.0", + "picomatch": "^2.0.4" + } + }, "aproba": { "version": "1.2.0", "resolved": "https://registry.npmjs.org/aproba/-/aproba-1.2.0.tgz", @@ -1108,7 +1181,7 @@ "array-flatten": { "version": "1.1.1", "resolved": "https://registry.npmjs.org/array-flatten/-/array-flatten-1.1.1.tgz", - "integrity": "sha512-PCVAQswWemu6UdxsDFFX/+gVeYqKAod3D3UVm91jHwynguOwAvYPhx8nNlM++NqRcK6CxxpUafjmhIdKiHibqg==" + "integrity": "sha1-ml9pkFGx5wczKPKgCJaLZOopVdI=" }, "array-includes": { "version": "3.1.1", @@ -1134,7 +1207,7 @@ "arrify": { "version": "2.0.1", "resolved": "https://registry.npmjs.org/arrify/-/arrify-2.0.1.tgz", - "integrity": "sha512-3duEwti880xqi4eAMN8AyR4a0ByT90zoYdLlevfrvU43vb0YZwZVfxOgxWrLXXXpyugL0hNZc9G6BiB5B3nUug==" + "integrity": "sha1-yWVekzHgq81YjSp8rX6ZVvZnAfo=" }, "asn1": { "version": "0.2.4", @@ -1147,12 +1220,12 @@ "assert-plus": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/assert-plus/-/assert-plus-1.0.0.tgz", - "integrity": "sha512-NfJ4UzBCcQGLDlQq7nHxH+tv3kyZ0hHQqF5BO6J7tNJeP5do1llPr8dZ8zHonfhAu0PHAdMkSo+8o0wxg9lZWw==" + "integrity": "sha1-8S4PPF13sLHN2RRpQuTpbB5N1SU=" }, "assertion-error": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/assertion-error/-/assertion-error-1.0.0.tgz", - "integrity": "sha512-g/gZV+G476cnmtYI+Ko9d5khxSoCSoom/EaNmmCfwpOvBXEJ18qwFrxfP1/CsIqk2no1sAKKwxndV0tP7ROOFQ==", + "integrity": "sha1-x/hUOP3UZrx8oWq5DIFRN5el0js=", "dev": true }, "astral-regex": { @@ -1164,7 +1237,7 @@ "async": { "version": "0.9.2", "resolved": "https://registry.npmjs.org/async/-/async-0.9.2.tgz", - "integrity": "sha512-l6ToIJIotphWahxxHyzK9bnLR6kM4jJIIgLShZeqLY7iboHoGkdgFl7W2/Ivi4SkMJYGKqW8vSuk0uKUj6qsSw==" + "integrity": "sha1-rqdNXmHB+JlhO/ZL2mbUx48v0X0=" }, "async-listener": { "version": "0.6.10", @@ -1185,12 +1258,12 @@ "asynckit": { "version": "0.4.0", "resolved": "https://registry.npmjs.org/asynckit/-/asynckit-0.4.0.tgz", - "integrity": "sha512-Oei9OH4tRh0YqU3GxhX79dM/mwVgvbZJaSNaRk+bshkj0S5cfHcgYakreBjrHwatXKbz+IoIdYLxrKim2MjW0Q==" + "integrity": "sha1-x57Zf380y48robyXkLzDZkdLS3k=" }, "aws-sign2": { "version": "0.7.0", "resolved": "https://registry.npmjs.org/aws-sign2/-/aws-sign2-0.7.0.tgz", - "integrity": "sha512-08kcGqnYf/YmjoRhfxyu+CLxBjUtHLXLXX/vUfx9l2LYzG3c1m61nrpyFUZI6zeS+Li/wWMMidD9KgrqtGq3mA==" + "integrity": "sha1-tG6JCTSpWR8tL2+G1+ap8bP+dqg=" }, "aws4": { "version": "1.10.0", @@ -1200,7 +1273,7 @@ "balanced-match": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/balanced-match/-/balanced-match-1.0.0.tgz", - "integrity": "sha512-9Y0g0Q8rmSt+H33DfKv7FOc3v+iRI+o1lbzt8jGcIosYW37IIW/2XVYq5NPdmaD5NQ59Nk26Kl/vZbwW9Fr8vg==" + "integrity": "sha1-ibTRmasr7kneFk6gK4nORi1xt2c=" }, "base64-js": { "version": "1.3.1", @@ -1210,17 +1283,17 @@ "base64id": { "version": "0.1.0", "resolved": "https://registry.npmjs.org/base64id/-/base64id-0.1.0.tgz", - "integrity": "sha512-DSjtfjhAsHl9J4OJj7e4+toV2zqxJrGwVd3CLlsCp8QmicvOn7irG0Mb8brOc/nur3SdO8lIbNlY1s1ZDJdUKQ==" + "integrity": "sha1-As4P3u4M709ACA4ec+g08LG/zj8=" }, "basic-auth-connect": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/basic-auth-connect/-/basic-auth-connect-1.0.0.tgz", - "integrity": "sha512-kiV+/DTgVro4aZifY/hwRwALBISViL5NP4aReaR2EVJEObpbUBHIkdJh/YpcoEiYt7nBodZ6U2ajZeZvSxUCCg==" + "integrity": "sha1-/bC0OWLKe0BFanwrtI/hc9otISI=" }, "bcrypt-pbkdf": { "version": "1.0.2", "resolved": "https://registry.npmjs.org/bcrypt-pbkdf/-/bcrypt-pbkdf-1.0.2.tgz", - "integrity": "sha512-qeFIXtP4MSoi6NLqO12WfqARWWuCKi2Rn/9hJLEmtB5yTNr9DqFWkJRCf2qShWzPeAMRnOgCrq0sg/KLv5ES9w==", + "integrity": "sha1-pDAdOJtqQ/m2f/PKEaP2Y342Dp4=", "requires": { "tweetnacl": "^0.14.3" } @@ -1230,10 +1303,16 @@ "resolved": "https://registry.npmjs.org/bignumber.js/-/bignumber.js-7.2.1.tgz", "integrity": "sha512-S4XzBk5sMB+Rcb/LNcpzXr57VRTxgAvaAEDAl1AwRx27j00hT84O6OkteE7u8UB3NuaaygCRrEpqox4uDOrbdQ==" }, + "binary-extensions": { + "version": "2.2.0", + "resolved": "https://registry.npmjs.org/binary-extensions/-/binary-extensions-2.2.0.tgz", + "integrity": "sha512-jDctJ/IVQbZoJykoeHbhXpOlNBqGNcwXJKJog42E5HDPUwQTSdjCHdihjj0DlnheQ7blbT6dHOafNAiS8ooQKA==", + "dev": true + }, "bindings": { "version": "1.5.0", "resolved": "https://registry.npmjs.org/bindings/-/bindings-1.5.0.tgz", - "integrity": "sha512-p2q/t/mhvuOj/UeLlV6566GD/guowlr0hHxClI0W9m7MWYkL1F0hLo+0Aexs9HSPCtR1SXQ0TD3MMKrXZajbiQ==", + "integrity": "sha1-EDU8npRTNLwFEabZCzj7x8nFBN8=", "requires": { "file-uri-to-path": "1.0.0" } @@ -1275,16 +1354,25 @@ "concat-map": "0.0.1" } }, + "braces": { + "version": "3.0.2", + "resolved": "https://registry.npmjs.org/braces/-/braces-3.0.2.tgz", + "integrity": "sha512-b8um+L1RzM3WDSzvhm6gIz1yfTbBt6YTlcEKAvsmqCZZFw46z626lVj9j1yEPW33H5H+lBQpZMP1k8l+78Ha0A==", + "dev": true, + "requires": { + "fill-range": "^7.0.1" + } + }, "browser-stdout": { - "version": "1.3.0", - "resolved": "https://registry.npmjs.org/browser-stdout/-/browser-stdout-1.3.0.tgz", - "integrity": "sha512-7Rfk377tpSM9TWBEeHs0FlDZGoAIei2V/4MdZJoFMBFAK6BqLpxAIUepGRHGdPFgGsLb02PXovC4qddyHvQqTg==", + "version": "1.3.1", + "resolved": "https://registry.npmjs.org/browser-stdout/-/browser-stdout-1.3.1.tgz", + "integrity": "sha512-qhAVI1+Av2X7qelOfAIYwXONood6XlZE/fXaBSmW/T5SzLAmCgzi+eiWE7fUvbHaeNBQH13UftjpXxsfLkMpgw==", "dev": true }, "buffer-equal-constant-time": { "version": "1.0.1", "resolved": "https://registry.npmjs.org/buffer-equal-constant-time/-/buffer-equal-constant-time-1.0.1.tgz", - "integrity": "sha512-zRpUiDwd/xk6ADqPMATG8vc9VPrkck7T07OIx0gnjmJAnHnTVXNQG3vfvWNuiZIkwu9KrKdA1iJKfsfTVxE6NA==" + "integrity": "sha1-+OcRMvf/5uAaXJaXpMbz5I1cyBk=" }, "buffer-from": { "version": "1.1.1", @@ -1294,12 +1382,12 @@ "builtin-modules": { "version": "3.1.0", "resolved": "https://registry.npmjs.org/builtin-modules/-/builtin-modules-3.1.0.tgz", - "integrity": "sha512-k0KL0aWZuBt2lrxrcASWDfwOLMnodeQjodT/1SxEQAXsHANgo6ZC/VEaSEHCXt7aSTZ4/4H5LKa+tBXmW7Vtvw==" + "integrity": "sha1-qtl8FRMet2tltQ7yCOdYTNdqdIQ=" }, "bunyan": { "version": "0.22.3", "resolved": "https://registry.npmjs.org/bunyan/-/bunyan-0.22.3.tgz", - "integrity": "sha512-v9dd5qmd6nJHEi7fiNo1fR2pMpE8AiB47Ap984p4iJKj+dEA69jSccmq6grFQn6pxIh0evvKpC5XO1SKfiaRoQ==", + "integrity": "sha1-ehncG0yMZF90AkGnQPIkUUfGfsI=", "dev": true, "requires": { "dtrace-provider": "0.2.8", @@ -1337,12 +1425,12 @@ "caseless": { "version": "0.12.0", "resolved": "https://registry.npmjs.org/caseless/-/caseless-0.12.0.tgz", - "integrity": "sha512-4tYFyifaFfGacoiObjJegolkwSU4xQNGbVgUiNYVUxbQ2x2lUsFvY4hVgVzGiIe6WLOPqycWXA40l+PWsxthUw==" + "integrity": "sha1-G2gcIf+EAzyCZUMJBolCDRhxUdw=" }, "chai": { "version": "1.9.2", "resolved": "https://registry.npmjs.org/chai/-/chai-1.9.2.tgz", - "integrity": "sha512-olRoaitftnzWHFEAza6MXR4w+FfZrOVyV7r7U/Z8ObJefCgL8IuWkAuASJjSXrpP9wvgoL8+1dB9RbMLc2FkNg==", + "integrity": "sha1-Pxog+CsLnXQ3V30k1vErGmnTtZA=", "dev": true, "requires": { "assertion-error": "1.0.0", @@ -1371,6 +1459,22 @@ "resolved": "https://registry.npmjs.org/charenc/-/charenc-0.0.2.tgz", "integrity": "sha1-wKHS86cJLgN3S/qD8UwPxXkKhmc=" }, + "chokidar": { + "version": "3.5.1", + "resolved": "https://registry.npmjs.org/chokidar/-/chokidar-3.5.1.tgz", + "integrity": "sha512-9+s+Od+W0VJJzawDma/gvBNQqkTiqYTWLuZoyAsivsI4AaWTCzHG06/TMjsf1cYe9Cb97UCEhjz7HvnPk2p/tw==", + "dev": true, + "requires": { + "anymatch": "~3.1.1", + "braces": "~3.0.2", + "fsevents": "~2.3.1", + "glob-parent": "~5.1.0", + "is-binary-path": "~2.1.0", + "is-glob": "~4.0.1", + "normalize-path": "~3.0.0", + "readdirp": "~3.5.0" + } + }, "chownr": { "version": "1.1.4", "resolved": "https://registry.npmjs.org/chownr/-/chownr-1.1.4.tgz", @@ -1430,7 +1534,7 @@ "cluster-key-slot": { "version": "1.1.0", "resolved": "https://registry.npmjs.org/cluster-key-slot/-/cluster-key-slot-1.1.0.tgz", - "integrity": "sha512-2Nii8p3RwAPiFwsnZvukotvow2rIHM+yQ6ZcBXGHdniadkYGZYiGmkHJIbZPIV9nfv7m/U1IPMVVcAhoWFeklw==" + "integrity": "sha1-MEdLKpgfsSFyaVgzBSvA0BM20Q0=" }, "code-point-at": { "version": "1.1.0", @@ -1515,12 +1619,12 @@ "concat-map": { "version": "0.0.1", "resolved": "https://registry.npmjs.org/concat-map/-/concat-map-0.0.1.tgz", - "integrity": "sha512-/Srv4dswyQNBfohGpz9o6Yb3Gz3SrUDqBH5rTuhGR7ahtlbYKnVxw2bCFMRljaA7EXHaXZ8wsHdodFvbkhKmqg==" + "integrity": "sha1-2Klr13/Wjfd5OnMDajug1UBdR3s=" }, "connect-redis": { "version": "2.5.1", "resolved": "https://registry.npmjs.org/connect-redis/-/connect-redis-2.5.1.tgz", - "integrity": "sha512-mnPCfN12SYcTw1yYRLejwv6WMmWpNtMkLeVIPzzAkC+u/bY8GxvMezG7wUKpMNoLmggS5xG2DWjEelggv6s5cw==", + "integrity": "sha1-6MCF227Gg7T8RXzzP5PD5+1vA/c=", "requires": { "debug": "^1.0.4", "redis": "^0.12.1" @@ -1529,7 +1633,7 @@ "debug": { "version": "1.0.5", "resolved": "https://registry.npmjs.org/debug/-/debug-1.0.5.tgz", - "integrity": "sha512-SIKSrp4+XqcUaNWhwaPJbLFnvSXPsZ4xBdH2WRK0Xo++UzMC4eepYghGAVhVhOwmfq3kqowqJ5w45R3pmYZnuA==", + "integrity": "sha1-9yQSF0MPmd7EwrRz6rkiKOh0wqw=", "requires": { "ms": "2.0.0" } @@ -1544,7 +1648,7 @@ "console-log-level": { "version": "1.4.1", "resolved": "https://registry.npmjs.org/console-log-level/-/console-log-level-1.4.1.tgz", - "integrity": "sha512-VZzbIORbP+PPcN/gg3DXClTLPLg5Slwd5fL2MIc+o1qZ4BXBvWyc6QxPk6T/Mkr6IVjRpoAGf32XxP3ZWMVRcQ==" + "integrity": "sha1-nFprue8e9lsFq6gwKLD/iUzfYwo=" }, "contains-path": { "version": "0.1.0", @@ -1622,7 +1726,7 @@ "core-util-is": { "version": "1.0.2", "resolved": "https://registry.npmjs.org/core-util-is/-/core-util-is-1.0.2.tgz", - "integrity": "sha512-3lqz5YjWTYnW6dlDa5TLaTCcShfar1e40rmcJVwCBJC6mWlFuj0eCHIElmG1g5kyuJ/GD+8Wn4FFCcz4gJPfaQ==" + "integrity": "sha1-tf1UIgqivFq1eqtxQMlAdUUDwac=" }, "cross-spawn": { "version": "6.0.5", @@ -1658,7 +1762,7 @@ "dashdash": { "version": "1.14.1", "resolved": "https://registry.npmjs.org/dashdash/-/dashdash-1.14.1.tgz", - "integrity": "sha512-jRFi8UDGo6j+odZiEpjazZaWqEal3w/basFjQHQEwVtZJGDpxbH1MeYluwCS8Xq5wmLJooDlMgvVarmWfGM44g==", + "integrity": "sha1-hTz6D3y+L+1d4gMmuN1YEDX24vA=", "requires": { "assert-plus": "^1.0.0" } @@ -1680,7 +1784,7 @@ "deep-eql": { "version": "0.1.3", "resolved": "https://registry.npmjs.org/deep-eql/-/deep-eql-0.1.3.tgz", - "integrity": "sha512-6sEotTRGBFiNcqVoeHwnfopbSpi5NbH1VWJmYCVkmxMmaVTT0bUTrNaGyBwhgP4MZL012W/mkzIn3Da+iDYweg==", + "integrity": "sha1-71WKyrjeJSBs1xOQbXTlaTDrafI=", "dev": true, "requires": { "type-detect": "0.1.1" @@ -1709,12 +1813,12 @@ "delay": { "version": "4.3.0", "resolved": "https://registry.npmjs.org/delay/-/delay-4.3.0.tgz", - "integrity": "sha512-Lwaf3zVFDMBop1yDuFZ19F9WyGcZcGacsbdlZtWjQmM50tOcMntm1njF/Nb/Vjij3KaSvCF+sEYGKrrjObu2NA==" + "integrity": "sha1-7+6/uPVFV5yzlrOnIkQ+yW0UxQ4=" }, "delayed-stream": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/delayed-stream/-/delayed-stream-1.0.0.tgz", - "integrity": "sha512-ZySD7Nf91aLB0RxL4KGrKHBXl7Eds1DAmEdcoVawXnLD7SDhpNgtuII2aAkg7a7QS41jxPSZ17p4VdGnMHk3MQ==" + "integrity": "sha1-3zrhmayt+31ECqrgsp4icrJOxhk=" }, "delegates": { "version": "1.0.0", @@ -1729,12 +1833,12 @@ "depd": { "version": "1.1.2", "resolved": "https://registry.npmjs.org/depd/-/depd-1.1.2.tgz", - "integrity": "sha512-7emPTl6Dpo6JRXOXjLRxck+FlLRX5847cLKEn00PLAgc3g2hTZZgr+e4c2v6QpSmLeFP3n5yUo7ft6avBK/5jQ==" + "integrity": "sha1-m81S4UwJd2PnSbJ0xDRu0uVgtak=" }, "destroy": { "version": "1.0.4", "resolved": "https://registry.npmjs.org/destroy/-/destroy-1.0.4.tgz", - "integrity": "sha512-3NdhDuEXnfun/z7x9GOElY49LoqVHoGScmOKwmxhsS8N5Y+Z8KyPPDnaSzqWgYt/ji4mqwfTS34Htrk0zPIXVg==" + "integrity": "sha1-l4hXRCxEdJ5CBmE+N5RiBYJqvYA=" }, "detect-libc": { "version": "1.0.3", @@ -1742,9 +1846,9 @@ "integrity": "sha1-+hN8S9aY7fVc1c0CrFWfkaTEups=" }, "diff": { - "version": "3.5.0", - "resolved": "https://registry.npmjs.org/diff/-/diff-3.5.0.tgz", - "integrity": "sha512-A46qtFgd+g7pDZinpnwiRJtxbC1hpgf0uzP3iG89scHk0AUC7A1TGxf5OiiOUv/JMZR8GOt8hL900hV0bOy5xA==", + "version": "4.0.2", + "resolved": "https://registry.npmjs.org/diff/-/diff-4.0.2.tgz", + "integrity": "sha512-58lmxKSA4BNyLz+HHMUzlOEpg09FV+ev6ZMe3vJihgdxzgcwZ8VoEEPmALCZG9LmqfVoNMMKpttIYTVG6uDY7A==", "dev": true }, "dlv": { @@ -1773,14 +1877,14 @@ "dtrace-provider": { "version": "0.2.8", "resolved": "https://registry.npmjs.org/dtrace-provider/-/dtrace-provider-0.2.8.tgz", - "integrity": "sha512-wufYnYt4ISHnT9MEiRgQ3trXuolt7mICTa/ckT+KYHR667K9H82lmI8KM7zKUJ8l5I343A34wJnvL++1TJn1iA==", + "integrity": "sha1-4kPxkhmqlfvw2PL/sH9b1k6U/iA=", "dev": true, "optional": true }, "duplexify": { "version": "3.7.1", "resolved": "https://registry.npmjs.org/duplexify/-/duplexify-3.7.1.tgz", - "integrity": "sha512-07z8uv2wMyS51kKhD1KsdXJg5WQ6t93RneqRxUHnskXVtlYYkLqM0gqStQZ3pj073g687jPCHrqNfCzawLYh5g==", + "integrity": "sha1-Kk31MX9sz9kfhtb9JdjYoQO4gwk=", "requires": { "end-of-stream": "^1.0.0", "inherits": "^2.0.1", @@ -1791,7 +1895,7 @@ "ecc-jsbn": { "version": "0.1.2", "resolved": "https://registry.npmjs.org/ecc-jsbn/-/ecc-jsbn-0.1.2.tgz", - "integrity": "sha512-eh9O+hwRHNbG4BLTjEl3nw044CkGm5X6LoaCf7LPp7UU8Qrt47JYNi6nPX8xjW97TKGKm1ouctg0QSpZe9qrnw==", + "integrity": "sha1-OoOpBOVDUyh4dMVkt1SThoSamMk=", "requires": { "jsbn": "~0.1.0", "safer-buffer": "^2.1.0" @@ -1800,7 +1904,7 @@ "ecdsa-sig-formatter": { "version": "1.0.11", "resolved": "https://registry.npmjs.org/ecdsa-sig-formatter/-/ecdsa-sig-formatter-1.0.11.tgz", - "integrity": "sha512-nagl3RYrbNv6kQkeJIpt6NJZy8twLB/2vtz6yN9Z4vRKHN4/QZJIEbqohALSgwKdnksuY3k5Addp5lg8sVoVcQ==", + "integrity": "sha1-rg8PothQRe8UqBfao86azQSJ5b8=", "requires": { "safe-buffer": "^5.0.1" } @@ -1808,7 +1912,7 @@ "ee-first": { "version": "1.1.1", "resolved": "https://registry.npmjs.org/ee-first/-/ee-first-1.1.1.tgz", - "integrity": "sha512-WMwm9LhRUo+WUaRN+vRuETqG89IgZphVSNkdFgeb6sS/E4OrDIN7t48CAewSHXc6C8lefD8KKfr5vY61brQlow==" + "integrity": "sha1-WQxhFWsK4vTwJVcyoViyZrxWsh0=" }, "emitter-listener": { "version": "1.1.2", @@ -1827,7 +1931,7 @@ "encodeurl": { "version": "1.0.2", "resolved": "https://registry.npmjs.org/encodeurl/-/encodeurl-1.0.2.tgz", - "integrity": "sha512-TPJXq8JqFaVYm2CWmPvnP2Iyo4ZSM7/QKcSmuMLDObfpH5fi7RUGmd/rTDf+rut/saiDiQEeVTNgAmJEdAOx0w==" + "integrity": "sha1-rT/0yG7C0CkyL1oCw6mmBslbP1k=" }, "end-of-stream": { "version": "1.4.4", @@ -1840,7 +1944,7 @@ "ent": { "version": "2.2.0", "resolved": "https://registry.npmjs.org/ent/-/ent-2.2.0.tgz", - "integrity": "sha512-GHrMyVZQWvTIdDtpiEXdHZnFQKzeO09apj8Cbl4pKWy4i0Oprcq17usfDt5aO63swf0JOeMWjWQE/LzgSRuWpA==" + "integrity": "sha1-6WQhkyWiHQX0RGai9obtbOX13R0=" }, "error-ex": { "version": "1.3.2", @@ -1881,15 +1985,21 @@ "is-symbol": "^1.0.2" } }, + "escalade": { + "version": "3.1.1", + "resolved": "https://registry.npmjs.org/escalade/-/escalade-3.1.1.tgz", + "integrity": "sha512-k0er2gUkLf8O0zKJiAhmkTnJlTvINGv7ygDNPbeIsX/TJjGJZHuh9B2UxbsaEkmlEo9MfhrSzmhIlhRlI2GXnw==", + "dev": true + }, "escape-html": { "version": "1.0.3", "resolved": "https://registry.npmjs.org/escape-html/-/escape-html-1.0.3.tgz", - "integrity": "sha512-NiSupZ4OeuGwr68lGIeym/ksIZMJodUGOSCZ/FSnTxcrekbvqrgdUxlJOMpijaKZVjAJrWrGs/6Jy8OMuyj9ow==" + "integrity": "sha1-Aljq5NPQwJdN4cFpGI7wBR0dGYg=" }, "escape-string-regexp": { "version": "1.0.5", "resolved": "https://registry.npmjs.org/escape-string-regexp/-/escape-string-regexp-1.0.5.tgz", - "integrity": "sha512-vbRorB5FUQWvla16U8R/qgaFIya2qGzwDrNmCZuYKrbdSUMG6I1ZCGQRefkRVhuOkIGVne7BQ35DSfo1qvJqFg==", + "integrity": "sha1-G2HAViGQqN/2rjuyzwIAyhMLhtQ=", "dev": true }, "eslint": { @@ -2234,12 +2344,12 @@ "etag": { "version": "1.8.1", "resolved": "https://registry.npmjs.org/etag/-/etag-1.8.1.tgz", - "integrity": "sha512-aIL5Fx7mawVa300al2BnEE4iNvo1qETxLrPI/o05L7z6go7fCw1J6EQmbK4FmJ2AS7kgVF/KEZWufBfdClMcPg==" + "integrity": "sha1-Qa4u62XvpiJorr/qg6x9eSmbCIc=" }, "event-target-shim": { "version": "5.0.1", "resolved": "https://registry.npmjs.org/event-target-shim/-/event-target-shim-5.0.1.tgz", - "integrity": "sha512-i/2XbnSz/uxRCU6+NdVJgKWDTM427+MqYbkQzD321DuCQJUqOuJKIA0IM2+W2xtYHdKOmZ4dR6fExsd4SXL+WQ==" + "integrity": "sha1-XU0+vflYPWOlMzzi3rdICrKwV4k=" }, "eventid": { "version": "1.0.0", @@ -2367,7 +2477,7 @@ "extsprintf": { "version": "1.3.0", "resolved": "https://registry.npmjs.org/extsprintf/-/extsprintf-1.3.0.tgz", - "integrity": "sha512-11Ndz7Nv+mvAC1j0ktTa7fAb0vLyGGX+rMHNBYQviQDGU0Hw7lhctJANqbPhu9nV9/izT/IntTgZ7Im/9LJs9g==" + "integrity": "sha1-lpGEQOMEGnpBT4xS48V06zw+HgU=" }, "fast-deep-equal": { "version": "3.1.3", @@ -2419,6 +2529,15 @@ "resolved": "https://registry.npmjs.org/file-uri-to-path/-/file-uri-to-path-1.0.0.tgz", "integrity": "sha512-0Zt+s3L7Vf1biwWZ29aARiVYLx7iMGnEUl9x33fbB/j3jR81u/O2LbqK+Bm1CDSNDKVtJ/YjwY7TUd5SkeLQLw==" }, + "fill-range": { + "version": "7.0.1", + "resolved": "https://registry.npmjs.org/fill-range/-/fill-range-7.0.1.tgz", + "integrity": "sha512-qOo9F+dMUmC2Lcb4BbVvnKJxTPjCm+RRpe4gDuGrzkL7mEVl/djYSu2OdQ2Pa302N4oqkSg9ir6jaLWJ2USVpQ==", + "dev": true, + "requires": { + "to-regex-range": "^5.0.1" + } + }, "finalhandler": { "version": "1.1.2", "resolved": "https://registry.npmjs.org/finalhandler/-/finalhandler-1.1.2.tgz", @@ -2447,6 +2566,12 @@ "resolved": "https://registry.npmjs.org/findit2/-/findit2-2.2.3.tgz", "integrity": "sha1-WKRmaX34piBc39vzlVNri9d3pfY=" }, + "flat": { + "version": "5.0.2", + "resolved": "https://registry.npmjs.org/flat/-/flat-5.0.2.tgz", + "integrity": "sha512-b6suED+5/3rTpUBdG1gupIl8MPFCAMA0QXwmljLhvCUKcUvdE4gWky9zpuGCcXHOsz4J9wPGNWq6OKpmIzz3hQ==", + "dev": true + }, "flat-cache": { "version": "2.0.1", "resolved": "https://registry.npmjs.org/flat-cache/-/flat-cache-2.0.1.tgz", @@ -2492,7 +2617,7 @@ "forever-agent": { "version": "0.6.1", "resolved": "https://registry.npmjs.org/forever-agent/-/forever-agent-0.6.1.tgz", - "integrity": "sha512-j0KLYPhm6zeac4lz3oJ3o65qvgQCcPubiyotZrXqEaG4hNagNYO8qdlUrX5vwqv9ohqeT/Z3j6+yW067yWWdUw==" + "integrity": "sha1-+8cfDEGt6zf5bFd60e1C2P2sypE=" }, "form-data": { "version": "2.3.3", @@ -2504,24 +2629,15 @@ "mime-types": "^2.1.12" } }, - "formatio": { - "version": "1.2.0", - "resolved": "https://registry.npmjs.org/formatio/-/formatio-1.2.0.tgz", - "integrity": "sha512-YAF05v8+XCxAyHOdiiAmHdgCVPrWO8X744fYIPtBciIorh5LndWfi1gjeJ16sTbJhzek9kd+j3YByhohtz5Wmg==", - "dev": true, - "requires": { - "samsam": "1.x" - } - }, "forwarded": { "version": "0.1.2", "resolved": "https://registry.npmjs.org/forwarded/-/forwarded-0.1.2.tgz", - "integrity": "sha512-Ua9xNhH0b8pwE3yRbFfXJvfdWF0UHNCdeyb2sbi9Ul/M+r3PTdrz7Cv4SCfZRMjmzEM9PhraqfZFbGTIg3OMyA==" + "integrity": "sha1-mMI9qxF1ZXuMBXPozszZGw/xjIQ=" }, "fresh": { "version": "0.5.2", "resolved": "https://registry.npmjs.org/fresh/-/fresh-0.5.2.tgz", - "integrity": "sha512-zJ2mQYM18rEFOudeV4GShTGIQ7RbzA7ozbU9I/XBpm7kqgMywgmylMwXHxZJmkVoYkna9d2pVXVXPdYTP9ej8Q==" + "integrity": "sha1-PYyt2Q2XZWn6g1qx+OSyOhBWBac=" }, "fs-minipass": { "version": "1.2.7", @@ -2534,7 +2650,14 @@ "fs.realpath": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/fs.realpath/-/fs.realpath-1.0.0.tgz", - "integrity": "sha512-OO0pH2lK6a0hZnAdau5ItzHPI6pUlvI7jMVnxUQRtw4owF2wk8lOSabtGDCTP4Ggrg2MbGnWO9X8K1t4+fGMDw==" + "integrity": "sha1-FQStJSMVjKpA20onh8sBQRmU6k8=" + }, + "fsevents": { + "version": "2.3.2", + "resolved": "https://registry.npmjs.org/fsevents/-/fsevents-2.3.2.tgz", + "integrity": "sha512-xiqMQR4xAeHTuB9uWm+fFRcIOgKBMiOBP+eXiyT7jsgVCq1bkVygt00oASowB7EdtpOHaaPgKt812P9ab+DDKA==", + "dev": true, + "optional": true }, "function-bind": { "version": "1.1.1", @@ -2611,7 +2734,7 @@ "getpass": { "version": "0.1.7", "resolved": "https://registry.npmjs.org/getpass/-/getpass-0.1.7.tgz", - "integrity": "sha512-0fzj9JxOLfJ+XGLhR8ze3unN0KZCgZwiSSDz168VERjK8Wl8kVSdcu2kspd4s4wtAa1y/qrVRiAA0WclVsu0ng==", + "integrity": "sha1-Xv+OPmhNVprkyysSgmBOi6YhSfo=", "requires": { "assert-plus": "^1.0.0" } @@ -2619,7 +2742,7 @@ "glob": { "version": "6.0.4", "resolved": "https://registry.npmjs.org/glob/-/glob-6.0.4.tgz", - "integrity": "sha512-MKZeRNyYZAVVVG1oZeLaWie1uweH40m9AZwIwxyPbTSX4hHrVYSzLg0Ro5Z5R7XKkIX+Cc6oD1rqeDJnwsB8/A==", + "integrity": "sha1-DwiGD2oVUSey+t1PnOJLGqtuTSI=", "optional": true, "requires": { "inflight": "^1.0.4", @@ -2792,10 +2915,16 @@ "integrity": "sha512-WjKPNJF79dtJAVniUlGGWHYGz2jWxT6VhN/4m1NdkbZ2nOsEF+cI1Edgql5zCRhs/VsQYRvrXctxktVXZUkixw==", "dev": true }, + "growl": { + "version": "1.10.5", + "resolved": "https://registry.npmjs.org/growl/-/growl-1.10.5.tgz", + "integrity": "sha512-qBr4OuELkhPenW6goKVXiv47US3clb3/IbuWF9KNKEijAy9oeHxU9IgzjvJhHkUzhaj7rOUD7+YGWqUjLp5oSA==", + "dev": true + }, "har-schema": { "version": "2.0.0", "resolved": "https://registry.npmjs.org/har-schema/-/har-schema-2.0.0.tgz", - "integrity": "sha512-Oqluz6zhGX8cyRaTQlFMPw80bSJVG2x/cFb8ZPhUILGgHka9SsokCCOQgpveePerqidZOrT14ipqfJb7ILcW5Q==" + "integrity": "sha1-qUwiJOvKwEeCoNkDVSHyRzW37JI=" }, "har-validator": { "version": "5.1.3", @@ -2850,9 +2979,9 @@ "integrity": "sha1-4Ob+aijPUROIVeCG0Wkedx3iqLk=" }, "he": { - "version": "1.1.1", - "resolved": "https://registry.npmjs.org/he/-/he-1.1.1.tgz", - "integrity": "sha512-z/GDPjlRMNOa2XJiB4em8wJpuuBfrFOlYKTZxtpkdr1uPdibHI8rYA3MY0KDObpVyaes0e/aunid/t88ZI2EKA==", + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/he/-/he-1.2.0.tgz", + "integrity": "sha512-F/1DnUGPopORZi0ni+CvrCgHQ5FyEAHRLSApuYWMmrbSwoN2Mn/7k+Gl38gJnR7yyDZk6WLXwiGod1JOWNDKGw==", "dev": true }, "hex2dec": { @@ -2914,7 +3043,7 @@ "http-signature": { "version": "1.2.0", "resolved": "https://registry.npmjs.org/http-signature/-/http-signature-1.2.0.tgz", - "integrity": "sha512-CAbnr6Rz4CYQkLYUtSNXxQPUH2gK8f3iWexVlsnMeD+GjlsQ0Xsy1cOX+mN3dtxYomRy21CiOzU8Uhw6OwncEQ==", + "integrity": "sha1-muzZJRFHcvPZW2WmCruPfBj7rOE=", "requires": { "assert-plus": "^1.0.0", "jsprim": "^1.2.2", @@ -2968,7 +3097,7 @@ "inflight": { "version": "1.0.6", "resolved": "https://registry.npmjs.org/inflight/-/inflight-1.0.6.tgz", - "integrity": "sha512-k92I/b08q4wvFscXCLvqfsHCrjrF7yiXsQuIVvVE7N82W3+aqpzuUdBbfhWcy/FZR3/4IgflMgKLOsvPDrGCJA==", + "integrity": "sha1-Sb1jMdfQLQwJvJEKEHW6gWW1bfk=", "requires": { "once": "^1.3.0", "wrappy": "1" @@ -2977,7 +3106,7 @@ "inherits": { "version": "2.0.3", "resolved": "https://registry.npmjs.org/inherits/-/inherits-2.0.3.tgz", - "integrity": "sha512-x00IRNXNy63jwGkJmzPigoySHbaqpNuzKbBOmzK+g2OdZpQ9w+sxCN+VSB3ja7IAge2OP2qpfxTjeNcyjmW1uw==" + "integrity": "sha1-Yzwsg+PaQqUC9SRmAiSA9CCCYd4=" }, "ini": { "version": "1.3.5", @@ -3113,6 +3242,15 @@ "integrity": "sha512-zz06S8t0ozoDXMG+ube26zeCTNXcKIPJZJi8hBrF4idCLms4CG9QtK7qBl1boi5ODzFpjswb5JPmHCbMpjaYzg==", "dev": true }, + "is-binary-path": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/is-binary-path/-/is-binary-path-2.1.0.tgz", + "integrity": "sha512-ZMERYes6pDydyuGidse7OsHxtbI7WVeUEozgR/g7rd0xUimYNlvZRE/K2MgZTjWy725IfelLeVcEM97mmtRGXw==", + "dev": true, + "requires": { + "binary-extensions": "^2.0.0" + } + }, "is-callable": { "version": "1.2.0", "resolved": "https://registry.npmjs.org/is-callable/-/is-callable-1.2.0.tgz", @@ -3146,11 +3284,23 @@ "is-extglob": "^2.1.1" } }, + "is-number": { + "version": "7.0.0", + "resolved": "https://registry.npmjs.org/is-number/-/is-number-7.0.0.tgz", + "integrity": "sha512-41Cifkg6e8TylSpdtTpeLVMqvSBEVzTttHvERD741+pnZ8ANv0004MRL43QKPDlK9cGvNp6NZWZUBlbGXYxxng==", + "dev": true + }, "is-obj": { "version": "2.0.0", "resolved": "https://registry.npmjs.org/is-obj/-/is-obj-2.0.0.tgz", "integrity": "sha512-drqDG3cbczxxEJRoOXcOjtdp1J/lyp1mNn0xaznRs8+muBhgQcrnbspox5X5fOw0HnMnbfDzvnEMEtqDEJEo8w==" }, + "is-plain-obj": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/is-plain-obj/-/is-plain-obj-2.1.0.tgz", + "integrity": "sha512-YWnfyRwxL/+SsrWYfOpUtz5b3YD+nyfkHvjbcanzk8zgyO4ASD67uVMRt8k5bM4lLMDnXfriRhOpemw+NfT1eA==", + "dev": true + }, "is-regex": { "version": "1.1.0", "resolved": "https://registry.npmjs.org/is-regex/-/is-regex-1.1.0.tgz", @@ -3188,12 +3338,12 @@ "is-typedarray": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/is-typedarray/-/is-typedarray-1.0.0.tgz", - "integrity": "sha512-cyA56iCMHAh5CdzjJIa4aohJyeO1YbwLi3Jc35MmRU6poroFjIGZzUzupGiRPOjgHg9TLu43xbpwXk523fMxKA==" + "integrity": "sha1-5HnICFjfDBsR3dppQPlgEfzaSpo=" }, "isarray": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/isarray/-/isarray-1.0.0.tgz", - "integrity": "sha512-VLghIWNM6ELQzo7zwmcg0NmTVyWKYjvIeM83yjp0wRDTmUnrM678fQbcKBo6n2CJEF0szoG//ytg+TKla89ALQ==" + "integrity": "sha1-u5NdSFgsuhaMBoNJV6VKPgcSTxE=" }, "isexe": { "version": "2.0.0", @@ -3204,7 +3354,7 @@ "isstream": { "version": "0.1.2", "resolved": "https://registry.npmjs.org/isstream/-/isstream-0.1.2.tgz", - "integrity": "sha512-Yljz7ffyPbrLpLngrMtZ7NduUgVvi6wG9RJ9IUcyCd59YQ911PBJphODUcbOVbqYfxe1wuYf/LJ8PauMRwsM/g==" + "integrity": "sha1-R+Y/evVa+m+S4VAOaQ64uFKcCZo=" }, "js-tokens": { "version": "4.0.0", @@ -3225,12 +3375,12 @@ "jsbn": { "version": "0.1.1", "resolved": "https://registry.npmjs.org/jsbn/-/jsbn-0.1.1.tgz", - "integrity": "sha512-UVU9dibq2JcFWxQPA6KCqj5O42VOmAY3zQUfEKxU0KpTGXwNoCjkX1e13eHNvw/xPynt6pU0rZ1htjWTNTSXsg==" + "integrity": "sha1-peZUwuWi3rXyAdls77yoDA7y9RM=" }, "json-bigint": { "version": "0.3.0", "resolved": "https://registry.npmjs.org/json-bigint/-/json-bigint-0.3.0.tgz", - "integrity": "sha512-u+c/u/F+JNPUekHCFyGVycRPyh9UHD5iUhSyIAn10kxbDTJxijwAbT6XHaONEOXuGGfmWUSroheXgHcml4gLgg==", + "integrity": "sha1-DM2RLEuCcNBfBW+9E4FLU9OCWx4=", "requires": { "bignumber.js": "^7.0.0" } @@ -3238,7 +3388,7 @@ "json-schema": { "version": "0.2.3", "resolved": "https://registry.npmjs.org/json-schema/-/json-schema-0.2.3.tgz", - "integrity": "sha512-a3xHnILGMtk+hDOqNwHzF6e2fNbiMrXZvxKQiEv2MlgQP+pjIOzqAmKYD2mDpXYE/44M7g+n9p2bKkYWDUcXCQ==" + "integrity": "sha1-tIDIkuWaLwWVTOcnvT8qTogvnhM=" }, "json-schema-traverse": { "version": "0.4.1", @@ -3254,7 +3404,7 @@ "json-stringify-safe": { "version": "5.0.1", "resolved": "https://registry.npmjs.org/json-stringify-safe/-/json-stringify-safe-5.0.1.tgz", - "integrity": "sha512-ZClg6AaYvamvYEE82d3Iyd3vSSIjQ+odgjaTzRuO3s7toCdFKczob2i0zCh7JE8kWn17yvAWhUVxvqGwUalsRA==" + "integrity": "sha1-Epai1Y/UXxmg9s4B1lcB4sc1tus=" }, "json5": { "version": "1.0.1", @@ -3276,7 +3426,7 @@ "jsprim": { "version": "1.4.1", "resolved": "https://registry.npmjs.org/jsprim/-/jsprim-1.4.1.tgz", - "integrity": "sha512-4Dj8Rf+fQ+/Pn7C5qeEX02op1WfOss3PKTE9Nsop3Dx+6UPxlm1dr/og7o2cRa5hNN07CACr4NFzRLtj/rjWog==", + "integrity": "sha1-MT5mvB5cwG5Di8G3SZwuXFastqI=", "requires": { "assert-plus": "1.0.0", "extsprintf": "1.3.0", @@ -3284,6 +3434,12 @@ "verror": "1.10.0" } }, + "just-extend": { + "version": "4.1.1", + "resolved": "https://registry.npmjs.org/just-extend/-/just-extend-4.1.1.tgz", + "integrity": "sha512-aWgeGFW67BP3e5181Ep1Fv2v8z//iBJfrvyTnq8wG86vEESwmonn1zPBJ0VfmT9CJq2FIT0VsETtrNFm2a+SHA==", + "dev": true + }, "levn": { "version": "0.3.0", "resolved": "https://registry.npmjs.org/levn/-/levn-0.3.0.tgz", @@ -3350,6 +3506,12 @@ "resolved": "https://registry.npmjs.org/lodash.flatten/-/lodash.flatten-4.4.0.tgz", "integrity": "sha1-8xwiIlqWMtK7+OSt2+8kCqdlph8=" }, + "lodash.get": { + "version": "4.4.2", + "resolved": "https://registry.npmjs.org/lodash.get/-/lodash.get-4.4.2.tgz", + "integrity": "sha1-LRd/ZS+jHpObRDjVNBSZ36OCXpk=", + "dev": true + }, "lodash.has": { "version": "4.5.2", "resolved": "https://registry.npmjs.org/lodash.has/-/lodash.has-4.5.2.tgz", @@ -3378,6 +3540,66 @@ "resolved": "https://registry.npmjs.org/log-driver/-/log-driver-1.2.7.tgz", "integrity": "sha512-U7KCmLdqsGHBLeWqYlFA0V0Sl6P08EE1ZrmA9cxjUE0WVqT9qnyVDPz1kzpFEP0jdJuFnasWIfSd7fsaNXkpbg==" }, + "log-symbols": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/log-symbols/-/log-symbols-4.0.0.tgz", + "integrity": "sha512-FN8JBzLx6CzeMrB0tg6pqlGU1wCrXW+ZXGH481kfsBqer0hToTIiHdjH4Mq8xJUbvATujKCvaREGWpGUionraA==", + "dev": true, + "requires": { + "chalk": "^4.0.0" + }, + "dependencies": { + "ansi-styles": { + "version": "4.3.0", + "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-4.3.0.tgz", + "integrity": "sha512-zbB9rCJAT1rbjiVDb2hqKFHNYLxgtk8NURxZ3IZwD3F6NtxbXZQCnnSi1Lkx+IDohdPlFp222wVALIheZJQSEg==", + "dev": true, + "requires": { + "color-convert": "^2.0.1" + } + }, + "chalk": { + "version": "4.1.0", + "resolved": "https://registry.npmjs.org/chalk/-/chalk-4.1.0.tgz", + "integrity": "sha512-qwx12AxXe2Q5xQ43Ac//I6v5aXTipYrSESdOgzrN+9XjgEpyjpKuvSGaN4qE93f7TQTlerQQ8S+EQ0EyDoVL1A==", + "dev": true, + "requires": { + "ansi-styles": "^4.1.0", + "supports-color": "^7.1.0" + } + }, + "color-convert": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/color-convert/-/color-convert-2.0.1.tgz", + "integrity": "sha512-RRECPsj7iu/xb5oKYcsFHSppFNnsj/52OVTRKb4zP5onXwVF3zVmmToNcOfGC+CRDpfK/U584fMg38ZHCaElKQ==", + "dev": true, + "requires": { + "color-name": "~1.1.4" + } + }, + "color-name": { + "version": "1.1.4", + "resolved": "https://registry.npmjs.org/color-name/-/color-name-1.1.4.tgz", + "integrity": "sha512-dOy+3AuW3a2wNbZHIuMZpTcgjGuLU/uBL/ubcZF9OXbDo8ff4O8yVp5Bf0efS8uEoYo5q4Fx7dY9OgQGXgAsQA==", + "dev": true + }, + "has-flag": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/has-flag/-/has-flag-4.0.0.tgz", + "integrity": "sha512-EykJT/Q1KjTWctppgIAgfSO0tKVuZUjhgMr17kqTumMl6Afv3EISleU7qZUzoXDFTAHTDC4NOoG/ZxU3EvlMPQ==", + "dev": true + }, + "supports-color": { + "version": "7.2.0", + "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-7.2.0.tgz", + "integrity": "sha512-qpCAvRl9stuOHveKsn7HncJRvv501qIacKzQlO/+Lwxc9+0q2wLyv4Dfvt80/DPn2pqOBsJdDiogXGR9+OvwRw==", + "dev": true, + "requires": { + "has-flag": "^4.0.0" + } + } + } + }, "logger-sharelatex": { "version": "2.2.0", "resolved": "https://registry.npmjs.org/logger-sharelatex/-/logger-sharelatex-2.2.0.tgz", @@ -3482,12 +3704,6 @@ } } }, - "lolex": { - "version": "1.6.0", - "resolved": "https://registry.npmjs.org/lolex/-/lolex-1.6.0.tgz", - "integrity": "sha512-/bpxDL56TG5LS5zoXxKqA6Ro5tkOS5M8cm/7yQcwLIKIcM2HR5fjjNCaIhJNv96SEk4hNGSafYMZK42Xv5fihQ==", - "dev": true - }, "long": { "version": "4.0.0", "resolved": "https://registry.npmjs.org/long/-/long-4.0.0.tgz", @@ -3544,12 +3760,12 @@ "media-typer": { "version": "0.3.0", "resolved": "https://registry.npmjs.org/media-typer/-/media-typer-0.3.0.tgz", - "integrity": "sha512-dq+qelQ9akHpcOl/gUVRTxVIOkAJ1wR3QAvb4RsVjS8oVoFjDGTc679wJYmUmknUF5HwMLOgb5O+a3KxfWapPQ==" + "integrity": "sha1-hxDXrwqmJvj/+hzgAWhUUmMlV0g=" }, "merge-descriptors": { "version": "1.0.1", "resolved": "https://registry.npmjs.org/merge-descriptors/-/merge-descriptors-1.0.1.tgz", - "integrity": "sha512-cCi6g3/Zr1iqQi6ySbseM1Xvooa98N0w31jzUYrXPX2xqObmFGHJ0tQ5u74H3mVh7wLouTseZyYIq39g8cNp1w==" + "integrity": "sha1-sAqqVW3YtEVoFQ7J0blT8/kMu2E=" }, "messageformat": { "version": "2.3.0", @@ -3577,7 +3793,7 @@ "methods": { "version": "1.1.2", "resolved": "https://registry.npmjs.org/methods/-/methods-1.1.2.tgz", - "integrity": "sha512-iclAHeNqNm68zFtnZ0e+1L2yUIdvzNoauKU4WBA3VvH/vPFieF7qfRlwUZU+DA9P9bPXIS90ulxoUoCH23sV2w==" + "integrity": "sha1-VSmk1nZUE07cxSZmVoNbD4Ua/O4=" }, "mime": { "version": "1.6.0", @@ -3614,7 +3830,7 @@ "minimist": { "version": "0.0.8", "resolved": "https://registry.npmjs.org/minimist/-/minimist-0.0.8.tgz", - "integrity": "sha512-miQKw5Hv4NS1Psg2517mV4e4dYNaO3++hjAvLOAzKqZ61rH8NS1SK+vbfBWZ5PY/Me/bEWhUwqMghEW5Fb9T7Q==" + "integrity": "sha1-hX/Kv8M5fSYluCKCYuhqp6ARsF0=" }, "minipass": { "version": "2.9.0", @@ -3643,54 +3859,128 @@ "mkdirp": { "version": "0.5.1", "resolved": "https://registry.npmjs.org/mkdirp/-/mkdirp-0.5.1.tgz", - "integrity": "sha512-SknJC52obPfGQPnjIkXbmA6+5H15E+fR+E4iR2oQ3zzCLbd7/ONua69R/Gw7AgkTLsRG+r5fzksYwWe1AgTyWA==", + "integrity": "sha1-MAV0OOrGz3+MR2fzhkjWaX11yQM=", "requires": { "minimist": "0.0.8" } }, "mocha": { - "version": "4.1.0", - "resolved": "https://registry.npmjs.org/mocha/-/mocha-4.1.0.tgz", - "integrity": "sha512-0RVnjg1HJsXY2YFDoTNzcc1NKhYuXKRrBAG2gDygmJJA136Cs2QlRliZG1mA0ap7cuaT30mw16luAeln+4RiNA==", + "version": "8.3.2", + "resolved": "https://registry.npmjs.org/mocha/-/mocha-8.3.2.tgz", + "integrity": "sha512-UdmISwr/5w+uXLPKspgoV7/RXZwKRTiTjJ2/AC5ZiEztIoOYdfKb19+9jNmEInzx5pBsCyJQzarAxqIGBNYJhg==", "dev": true, "requires": { - "browser-stdout": "1.3.0", - "commander": "2.11.0", - "debug": "3.1.0", - "diff": "3.3.1", - "escape-string-regexp": "1.0.5", - "glob": "7.1.2", - "growl": "1.10.3", - "he": "1.1.1", - "mkdirp": "0.5.1", - "supports-color": "4.4.0" + "@ungap/promise-all-settled": "1.1.2", + "ansi-colors": "4.1.1", + "browser-stdout": "1.3.1", + "chokidar": "3.5.1", + "debug": "4.3.1", + "diff": "5.0.0", + "escape-string-regexp": "4.0.0", + "find-up": "5.0.0", + "glob": "7.1.6", + "growl": "1.10.5", + "he": "1.2.0", + "js-yaml": "4.0.0", + "log-symbols": "4.0.0", + "minimatch": "3.0.4", + "ms": "2.1.3", + "nanoid": "3.1.20", + "serialize-javascript": "5.0.1", + "strip-json-comments": "3.1.1", + "supports-color": "8.1.1", + "which": "2.0.2", + "wide-align": "1.1.3", + "workerpool": "6.1.0", + "yargs": "16.2.0", + "yargs-parser": "20.2.4", + "yargs-unparser": "2.0.0" }, "dependencies": { - "commander": { - "version": "2.11.0", - "resolved": "https://registry.npmjs.org/commander/-/commander-2.11.0.tgz", - "integrity": "sha512-b0553uYA5YAEGgyYIGYROzKQ7X5RAqedkfjiZxwi0kL1g3bOaBNNZfYkzt/CL0umgD5wc9Jec2FbB98CjkMRvQ==", + "ansi-styles": { + "version": "4.3.0", + "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-4.3.0.tgz", + "integrity": "sha512-zbB9rCJAT1rbjiVDb2hqKFHNYLxgtk8NURxZ3IZwD3F6NtxbXZQCnnSi1Lkx+IDohdPlFp222wVALIheZJQSEg==", + "dev": true, + "requires": { + "color-convert": "^2.0.1" + } + }, + "argparse": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/argparse/-/argparse-2.0.1.tgz", + "integrity": "sha512-8+9WqebbFzpX9OR+Wa6O29asIogeRMzcGtAINdpMHHyAg10f05aSFVBbcEqGf/PXw1EjAZ+q2/bEBg3DvurK3Q==", + "dev": true + }, + "cliui": { + "version": "7.0.4", + "resolved": "https://registry.npmjs.org/cliui/-/cliui-7.0.4.tgz", + "integrity": "sha512-OcRE68cOsVMXp1Yvonl/fzkQOyjLSu/8bhPDfQt0e0/Eb283TKP20Fs2MqoPsr9SwA595rRCA+QMzYc9nBP+JQ==", + "dev": true, + "requires": { + "string-width": "^4.2.0", + "strip-ansi": "^6.0.0", + "wrap-ansi": "^7.0.0" + } + }, + "color-convert": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/color-convert/-/color-convert-2.0.1.tgz", + "integrity": "sha512-RRECPsj7iu/xb5oKYcsFHSppFNnsj/52OVTRKb4zP5onXwVF3zVmmToNcOfGC+CRDpfK/U584fMg38ZHCaElKQ==", + "dev": true, + "requires": { + "color-name": "~1.1.4" + } + }, + "color-name": { + "version": "1.1.4", + "resolved": "https://registry.npmjs.org/color-name/-/color-name-1.1.4.tgz", + "integrity": "sha512-dOy+3AuW3a2wNbZHIuMZpTcgjGuLU/uBL/ubcZF9OXbDo8ff4O8yVp5Bf0efS8uEoYo5q4Fx7dY9OgQGXgAsQA==", "dev": true }, "debug": { - "version": "3.1.0", - "resolved": "https://registry.npmjs.org/debug/-/debug-3.1.0.tgz", - "integrity": "sha512-OX8XqP7/1a9cqkxYw2yXss15f26NKWBpDXQd0/uK/KPqdQhxbPa994hnzjcE2VqQpDslf55723cKPUOGSmMY3g==", + "version": "4.3.1", + "resolved": "https://registry.npmjs.org/debug/-/debug-4.3.1.tgz", + "integrity": "sha512-doEwdvm4PCeK4K3RQN2ZC2BYUBaxwLARCqZmMjtF8a51J2Rb0xpVloFRnCODwqjpwnAoao4pelN8l3RJdv3gRQ==", "dev": true, "requires": { - "ms": "2.0.0" + "ms": "2.1.2" + }, + "dependencies": { + "ms": { + "version": "2.1.2", + "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.2.tgz", + "integrity": "sha512-sGkPx+VjMtmA6MX27oA4FBFELFCZZ4S4XqeGOXCv68tT+jb3vk/RyaKWP0PTKyWtmLSM0b+adUTEvbs1PEaH2w==", + "dev": true + } } }, "diff": { - "version": "3.3.1", - "resolved": "https://registry.npmjs.org/diff/-/diff-3.3.1.tgz", - "integrity": "sha512-MKPHZDMB0o6yHyDryUOScqZibp914ksXwAMYMTHj6KO8UeKsRYNJD3oNCKjTqZon+V488P7N/HzXF8t7ZR95ww==", + "version": "5.0.0", + "resolved": "https://registry.npmjs.org/diff/-/diff-5.0.0.tgz", + "integrity": "sha512-/VTCrvm5Z0JGty/BWHljh+BAiw3IK+2j87NGMu8Nwc/f48WoDAC395uomO9ZD117ZOBaHmkX1oyLvkVM/aIT3w==", "dev": true }, + "escape-string-regexp": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/escape-string-regexp/-/escape-string-regexp-4.0.0.tgz", + "integrity": "sha512-TtpcNJ3XAzx3Gq8sWRzJaVajRs0uVxA2YAkdb1jm2YkPz4G6egUFAyA3n5vtEIZefPk5Wa4UXbKuS5fKkJWdgA==", + "dev": true + }, + "find-up": { + "version": "5.0.0", + "resolved": "https://registry.npmjs.org/find-up/-/find-up-5.0.0.tgz", + "integrity": "sha512-78/PXT1wlLLDgTzDs7sjq9hzz0vXD+zn+7wypEe4fXQxCmdmqfGsEPQxmiCSQI3ajFV91bVSsvNtrJRiW6nGng==", + "dev": true, + "requires": { + "locate-path": "^6.0.0", + "path-exists": "^4.0.0" + } + }, "glob": { - "version": "7.1.2", - "resolved": "https://registry.npmjs.org/glob/-/glob-7.1.2.tgz", - "integrity": "sha512-MJTUg1kjuLeQCJ+ccE4Vpa6kKVXkPYJ2mOCQyUuKLcLQsdrMCpBPUi8qVE6+YuaJkozeA9NusTAw3hLr8Xe5EQ==", + "version": "7.1.6", + "resolved": "https://registry.npmjs.org/glob/-/glob-7.1.6.tgz", + "integrity": "sha512-LwaxwyZ72Lk7vZINtNNrywX0ZuLyStrdDtabefZKAY5ZGJhVtgdznluResxNmPitE0SAO+O26sWTHeKSI2wMBA==", "dev": true, "requires": { "fs.realpath": "^1.0.0", @@ -3701,26 +3991,130 @@ "path-is-absolute": "^1.0.0" } }, - "growl": { - "version": "1.10.3", - "resolved": "https://registry.npmjs.org/growl/-/growl-1.10.3.tgz", - "integrity": "sha512-hKlsbA5Vu3xsh1Cg3J7jSmX/WaW6A5oBeqzM88oNbCRQFz+zUaXm6yxS4RVytp1scBoJzSYl4YAEOQIt6O8V1Q==", + "has-flag": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/has-flag/-/has-flag-4.0.0.tgz", + "integrity": "sha512-EykJT/Q1KjTWctppgIAgfSO0tKVuZUjhgMr17kqTumMl6Afv3EISleU7qZUzoXDFTAHTDC4NOoG/ZxU3EvlMPQ==", "dev": true }, - "has-flag": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/has-flag/-/has-flag-2.0.0.tgz", - "integrity": "sha512-P+1n3MnwjR/Epg9BBo1KT8qbye2g2Ou4sFumihwt6I4tsUX7jnLcX4BTOSKg/B1ZrIYMN9FcEnG4x5a7NB8Eng==", + "js-yaml": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/js-yaml/-/js-yaml-4.0.0.tgz", + "integrity": "sha512-pqon0s+4ScYUvX30wxQi3PogGFAlUyH0awepWvwkj4jD4v+ova3RiYw8bmA6x2rDrEaj8i/oWKoRxpVNW+Re8Q==", + "dev": true, + "requires": { + "argparse": "^2.0.1" + } + }, + "locate-path": { + "version": "6.0.0", + "resolved": "https://registry.npmjs.org/locate-path/-/locate-path-6.0.0.tgz", + "integrity": "sha512-iPZK6eYjbxRu3uB4/WZ3EsEIMJFMqAoopl3R+zuq0UjcAm/MO6KCweDgPfP3elTztoKP3KtnVHxTn2NHBSDVUw==", + "dev": true, + "requires": { + "p-locate": "^5.0.0" + } + }, + "ms": { + "version": "2.1.3", + "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.3.tgz", + "integrity": "sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA==", + "dev": true + }, + "p-limit": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/p-limit/-/p-limit-3.1.0.tgz", + "integrity": "sha512-TYOanM3wGwNGsZN2cVTYPArw454xnXj5qmWF1bEoAc4+cU/ol7GVh7odevjp1FNHduHc3KZMcFduxU5Xc6uJRQ==", + "dev": true, + "requires": { + "yocto-queue": "^0.1.0" + } + }, + "p-locate": { + "version": "5.0.0", + "resolved": "https://registry.npmjs.org/p-locate/-/p-locate-5.0.0.tgz", + "integrity": "sha512-LaNjtRWUBY++zB5nE/NwcaoMylSPk+S+ZHNB1TzdbMJMny6dynpAGt7X/tl/QYq3TIeE6nxHppbo2LGymrG5Pw==", + "dev": true, + "requires": { + "p-limit": "^3.0.2" + } + }, + "path-exists": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/path-exists/-/path-exists-4.0.0.tgz", + "integrity": "sha512-ak9Qy5Q7jYb2Wwcey5Fpvg2KoAc/ZIhLSLOSBmRmygPsGwkVVt0fZa0qrtMz+m6tJTAHfZQ8FnmB4MG4LWy7/w==", + "dev": true + }, + "strip-ansi": { + "version": "6.0.0", + "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-6.0.0.tgz", + "integrity": "sha512-AuvKTrTfQNYNIctbR1K/YGTR1756GycPsg7b9bdV9Duqur4gv6aKqHXah67Z8ImS7WEz5QVcOtlfW2rZEugt6w==", + "dev": true, + "requires": { + "ansi-regex": "^5.0.0" + } + }, + "strip-json-comments": { + "version": "3.1.1", + "resolved": "https://registry.npmjs.org/strip-json-comments/-/strip-json-comments-3.1.1.tgz", + "integrity": "sha512-6fPc+R4ihwqP6N/aIv2f1gMH8lOVtWQHoqC4yK6oSDVVocumAsfCqjkXnqiYMhmMwS/mEHLp7Vehlt3ql6lEig==", "dev": true }, "supports-color": { - "version": "4.4.0", - "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-4.4.0.tgz", - "integrity": "sha512-rKC3+DyXWgK0ZLKwmRsrkyHVZAjNkfzeehuFWdGGcqGDTZFH73+RH6S/RDAAxl9GusSjZSUWYLmT9N5pzXFOXQ==", + "version": "8.1.1", + "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-8.1.1.tgz", + "integrity": "sha512-MpUEN2OodtUzxvKQl72cUF7RQ5EiHsGvSsVG0ia9c5RbWGL2CI4C7EpPS8UTBIplnlzZiNuV56w+FuNxy3ty2Q==", "dev": true, "requires": { - "has-flag": "^2.0.0" + "has-flag": "^4.0.0" } + }, + "which": { + "version": "2.0.2", + "resolved": "https://registry.npmjs.org/which/-/which-2.0.2.tgz", + "integrity": "sha512-BLI3Tl1TW3Pvl70l3yq3Y64i+awpwXqsGBYWkkqMtnbXgrMD+yj7rhW0kuEDxzJaYXGjEW5ogapKNMEKNMjibA==", + "dev": true, + "requires": { + "isexe": "^2.0.0" + } + }, + "wrap-ansi": { + "version": "7.0.0", + "resolved": "https://registry.npmjs.org/wrap-ansi/-/wrap-ansi-7.0.0.tgz", + "integrity": "sha512-YVGIj2kamLSTxw6NsZjoBxfSwsn0ycdesmc4p+Q21c5zPuZ1pl+NfxVdxPtdHvmNVOQ6XSYG4AUtyt/Fi7D16Q==", + "dev": true, + "requires": { + "ansi-styles": "^4.0.0", + "string-width": "^4.1.0", + "strip-ansi": "^6.0.0" + } + }, + "y18n": { + "version": "5.0.5", + "resolved": "https://registry.npmjs.org/y18n/-/y18n-5.0.5.tgz", + "integrity": "sha512-hsRUr4FFrvhhRH12wOdfs38Gy7k2FFzB9qgN9v3aLykRq0dRcdcpz5C9FxdS2NuhOrI/628b/KSTJ3rwHysYSg==", + "dev": true + }, + "yargs": { + "version": "16.2.0", + "resolved": "https://registry.npmjs.org/yargs/-/yargs-16.2.0.tgz", + "integrity": "sha512-D1mvvtDG0L5ft/jGWkLpG1+m0eQxOfaBvTNELraWj22wSVUMWxZUvYgJYcKh6jGGIkJFhH4IZPQhR4TKpc8mBw==", + "dev": true, + "requires": { + "cliui": "^7.0.2", + "escalade": "^3.1.1", + "get-caller-file": "^2.0.5", + "require-directory": "^2.1.1", + "string-width": "^4.2.0", + "y18n": "^5.0.5", + "yargs-parser": "^20.2.2" + } + }, + "yargs-parser": { + "version": "20.2.4", + "resolved": "https://registry.npmjs.org/yargs-parser/-/yargs-parser-20.2.4.tgz", + "integrity": "sha512-WOkpgNhPTlE73h4VFAFsOnomJVaovO8VqLDzy5saChRBFQFBoMYirowyW+Q9HB4HFF4Z7VZTiG3iSzJJA29yRA==", + "dev": true } } }, @@ -3738,7 +4132,7 @@ "ms": { "version": "2.0.0", "resolved": "https://registry.npmjs.org/ms/-/ms-2.0.0.tgz", - "integrity": "sha512-Tpp60P6IUJDTuOq/5Z8cdskzJujfwqfOTkrwIwj7IRISpnkJnT6SyJ4PCPnGMoFjC9ddhal5KVIYtAt97ix05A==" + "integrity": "sha1-VgiurfwAvmwpAd9fmGF4jeDVl8g=" }, "mute-stream": { "version": "0.0.8", @@ -3749,7 +4143,7 @@ "mv": { "version": "2.1.1", "resolved": "https://registry.npmjs.org/mv/-/mv-2.1.1.tgz", - "integrity": "sha512-at/ZndSy3xEGJ8i0ygALh8ru9qy7gWW1cmkaqBN29JmMlIvM//MEO9y1sk/avxuwnPcfhkejkLsuPxH81BrkSg==", + "integrity": "sha1-rmzg1vbV4KT32JN5jQPB6pVZtqI=", "optional": true, "requires": { "mkdirp": "~0.5.1", @@ -3757,10 +4151,10 @@ "rimraf": "~2.4.0" } }, - "native-promise-only": { - "version": "0.8.1", - "resolved": "https://registry.npmjs.org/native-promise-only/-/native-promise-only-0.8.1.tgz", - "integrity": "sha512-zkVhZUA3y8mbz652WrL5x0fB0ehrBkulWT3TomAQ9iDtyXZvzKeEA6GPxAItBYeNYl5yngKRX612qHOhvMkDeg==", + "nanoid": { + "version": "3.1.20", + "resolved": "https://registry.npmjs.org/nanoid/-/nanoid-3.1.20.tgz", + "integrity": "sha512-a1cQNyczgKbLX9jwbS/+d7W8fX/RfgYR7lVWwWOGIPNgK2m0MWvrGF6/m4kk6U3QcFMnZf3RIhL0v2Jgh/0Uxw==", "dev": true }, "natural-compare": { @@ -3772,7 +4166,7 @@ "ncp": { "version": "2.0.0", "resolved": "https://registry.npmjs.org/ncp/-/ncp-2.0.0.tgz", - "integrity": "sha512-zIdGUrPRFTUELUvr3Gmc7KZ2Sw/h1PiVM0Af/oHB6zgnV1ikqSfRk+TOufi79aHYCW3NiOXmr1BP5nWbzojLaA==", + "integrity": "sha1-GVoh1sRuNh0vsSgbo4uR6d9727M=", "optional": true }, "needle": { @@ -3811,6 +4205,36 @@ "integrity": "sha512-1nh45deeb5olNY7eX82BkPO7SSxR5SSYJiPTrTdFUVYwAl8CKMA5N9PjTYkHiRjisVcxcQ1HXdLhx2qxxJzLNQ==", "dev": true }, + "nise": { + "version": "4.1.0", + "resolved": "https://registry.npmjs.org/nise/-/nise-4.1.0.tgz", + "integrity": "sha512-eQMEmGN/8arp0xsvGoQ+B1qvSkR73B1nWSCh7nOt5neMCtwcQVYQGdzQMhcNscktTsWB54xnlSQFzOAPJD8nXA==", + "dev": true, + "requires": { + "@sinonjs/commons": "^1.7.0", + "@sinonjs/fake-timers": "^6.0.0", + "@sinonjs/text-encoding": "^0.7.1", + "just-extend": "^4.0.2", + "path-to-regexp": "^1.7.0" + }, + "dependencies": { + "isarray": { + "version": "0.0.1", + "resolved": "https://registry.npmjs.org/isarray/-/isarray-0.0.1.tgz", + "integrity": "sha1-ihis/Kmo9Bd+Cav8YDiTmwXR7t8=", + "dev": true + }, + "path-to-regexp": { + "version": "1.8.0", + "resolved": "https://registry.npmjs.org/path-to-regexp/-/path-to-regexp-1.8.0.tgz", + "integrity": "sha512-n43JRhlUKUAlibEJhPeir1ncUID16QnEjNpwzNdO3Lm4ywrBpBZ5oLD0I6br9evr1Y9JTqwRtAh7JLoOzAQdVA==", + "dev": true, + "requires": { + "isarray": "0.0.1" + } + } + } + }, "node-fetch": { "version": "2.6.1", "resolved": "https://registry.npmjs.org/node-fetch/-/node-fetch-2.6.1.tgz", @@ -3903,6 +4327,12 @@ } } }, + "normalize-path": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/normalize-path/-/normalize-path-3.0.0.tgz", + "integrity": "sha512-6eZs5Ls3WtCisHWp9S2GUy8dqkpGi4BVSz3GaqiE6ezub0512ESztXUwUB6C6IKbQkY2Pnb/mD4WYojCRwcwLA==", + "dev": true + }, "npm-bundled": { "version": "1.1.1", "resolved": "https://registry.npmjs.org/npm-bundled/-/npm-bundled-1.1.1.tgz", @@ -3991,7 +4421,7 @@ "on-finished": { "version": "2.3.0", "resolved": "https://registry.npmjs.org/on-finished/-/on-finished-2.3.0.tgz", - "integrity": "sha512-ikqdkGAAyf/X/gPhXGvfgAytDZtDbr+bkNUJ0N9h5MI/dmdgCs3l6hoHrcUv41sRKew3jIwrp4qQDXiK99Utww==", + "integrity": "sha1-IPEzZIGwg811M3mSoWlxqi2QaUc=", "requires": { "ee-first": "1.1.1" } @@ -4004,7 +4434,7 @@ "once": { "version": "1.4.0", "resolved": "https://registry.npmjs.org/once/-/once-1.4.0.tgz", - "integrity": "sha512-lNaJgI+2Q5URQBkccEKHTQOPaXdUxnZZElQTZY0MFUAuaEqe1E+Nyvgdz/aIyNi6Z9MzO5dv1H8n58/GELp3+w==", + "integrity": "sha1-WDsap3WWHUsROsF9nFC6753Xa9E=", "requires": { "wrappy": "1" } @@ -4094,7 +4524,7 @@ "p-try": { "version": "2.2.0", "resolved": "https://registry.npmjs.org/p-try/-/p-try-2.2.0.tgz", - "integrity": "sha512-R4nPAVTAU0B9D35/Gk3uJf/7XYbQcyohSKdvAxIRSNghFl4e71hVoGnBNQz9cWaXxO2I10KTC+3jMdvvoKw6dQ==", + "integrity": "sha1-yyhoVA4xPWHeWPr741zpAE1VQOY=", "dev": true }, "parent-module": { @@ -4118,7 +4548,7 @@ "parse-ms": { "version": "2.1.0", "resolved": "https://registry.npmjs.org/parse-ms/-/parse-ms-2.1.0.tgz", - "integrity": "sha512-kHt7kzLoS9VBZfUsiKjv43mr91ea+U05EyKkEtqp7vNbHxmaVuEqN7XxeEVnGrMtYOAxGrDElSi96K7EgO1zCA==" + "integrity": "sha1-NIVlp1PUOR+lJAKZVrFyy3dTCX0=" }, "parseurl": { "version": "1.3.3", @@ -4134,7 +4564,7 @@ "path-is-absolute": { "version": "1.0.1", "resolved": "https://registry.npmjs.org/path-is-absolute/-/path-is-absolute-1.0.1.tgz", - "integrity": "sha512-AVbw3UJ2e9bq64vSaS9Am0fje1Pa8pbGqTTsmXfaIiMpnr5DlDhfJOuLj9Sf95ZPVDAUerDfEk88MPmPe7UCQg==" + "integrity": "sha1-F0uSaHNVNP+8es5r9TpanhtcX18=" }, "path-is-inside": { "version": "1.0.2", @@ -4156,7 +4586,7 @@ "path-to-regexp": { "version": "0.1.7", "resolved": "https://registry.npmjs.org/path-to-regexp/-/path-to-regexp-0.1.7.tgz", - "integrity": "sha512-5DFkuoqlv1uYQKxy8omFBeJPQcdoE07Kv2sferDCrAq1ohOU+MSDswDIbnx3YAM60qIOnYa53wBhXW0EbMonrQ==" + "integrity": "sha1-32BBeABfUi8V60SQ5yR6G/qmf4w=" }, "path-type": { "version": "2.0.0", @@ -4178,7 +4608,13 @@ "performance-now": { "version": "2.1.0", "resolved": "https://registry.npmjs.org/performance-now/-/performance-now-2.1.0.tgz", - "integrity": "sha512-7EAHlyLHI56VEIdK57uwHdHKIaAGbnXPiw0yWbarQZOKaKpvUIgW0jWRVLiatnM+XXlSwsanIBH/hzGMJulMow==" + "integrity": "sha1-Ywn04OX6kT7BxpMHrjZLSzd8nns=" + }, + "picomatch": { + "version": "2.2.2", + "resolved": "https://registry.npmjs.org/picomatch/-/picomatch-2.2.2.tgz", + "integrity": "sha512-q0M/9eZHzmr0AulXyPwNfZjtwZ/RBZlbN3K3CErVrk50T2ASYI7Bye0EvekFY3IP1Nt2DHu0re+V2ZHIpMkuWg==", + "dev": true }, "pkg-dir": { "version": "2.0.0", @@ -5004,7 +5440,16 @@ "random-bytes": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/random-bytes/-/random-bytes-1.0.0.tgz", - "integrity": "sha512-iv7LhNVO047HzYR3InF6pUcUsPQiHTM1Qal51DcGSuZFBil1aBBWG5eHPNek7bvILMaYJ/8RU1e8w1AMdHmLQQ==" + "integrity": "sha1-T2ih3Arli9P7lYSMMDJNt11kNgs=" + }, + "randombytes": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/randombytes/-/randombytes-2.1.0.tgz", + "integrity": "sha512-vYl3iOX+4CKUWuxGi9Ukhie6fsqXqS9FE2Zaic4tNFD2N2QQaXOMFbuKK4QmDHC0JO6B1Zp41J0LpT0oR68amQ==", + "dev": true, + "requires": { + "safe-buffer": "^5.1.0" + } }, "range-parser": { "version": "1.2.1", @@ -5099,15 +5544,24 @@ "util-deprecate": "~1.0.1" } }, + "readdirp": { + "version": "3.5.0", + "resolved": "https://registry.npmjs.org/readdirp/-/readdirp-3.5.0.tgz", + "integrity": "sha512-cMhu7c/8rdhkHXWsY+osBhfSy0JikwpHK/5+imo+LpeasTF8ouErHrlYkwT0++njiyuDvc7OFY5T3ukvZ8qmFQ==", + "dev": true, + "requires": { + "picomatch": "^2.2.1" + } + }, "redis": { "version": "0.12.1", "resolved": "https://registry.npmjs.org/redis/-/redis-0.12.1.tgz", - "integrity": "sha512-DtqxdmgmVAO7aEyxaXBiUTvhQPOYznTIvmPzs9AwWZqZywM50JlFxQjFhicI+LVbaun7uwfO3izuvc1L8NlPKQ==" + "integrity": "sha1-ZN92rQ/IrOuuvSoGReikj6xJGF4=" }, "redis-commands": { "version": "1.5.0", "resolved": "https://registry.npmjs.org/redis-commands/-/redis-commands-1.5.0.tgz", - "integrity": "sha512-6KxamqpZ468MeQC3bkWmCB1fp56XL64D4Kf0zJSwDZbVLLm7KFkoIcHrgRvQ+sk8dnhySs7+yBg94yIkAK7aJg==" + "integrity": "sha1-gNLiBpj+aI8icSf/nlFkp90X54U=" }, "redis-errors": { "version": "1.2.0", @@ -5203,7 +5657,7 @@ "require-like": { "version": "0.1.2", "resolved": "https://registry.npmjs.org/require-like/-/require-like-0.1.2.tgz", - "integrity": "sha512-oyrU88skkMtDdauHDuKVrgR+zuItqr6/c//FXzvmxRGMexSDc6hNvJInGW3LL46n+8b50RykrvwSUIIQH2LQ5A==", + "integrity": "sha1-rW8wwTvs15cBDEaK+ndcDAprR/o=", "dev": true }, "require-main-filename": { @@ -5269,7 +5723,7 @@ "rimraf": { "version": "2.4.5", "resolved": "https://registry.npmjs.org/rimraf/-/rimraf-2.4.5.tgz", - "integrity": "sha512-J5xnxTyqaiw06JjMftq7L9ouA448dw/E7dKghkP9WpKNuwmARNNg+Gk8/u5ryb9N/Yo2+z3MCwuqFK/+qPOPfQ==", + "integrity": "sha1-7nEM5dk6j9uFb7Xqj/Di11k0sto=", "optional": true, "requires": { "glob": "^6.0.1" @@ -5306,16 +5760,10 @@ "resolved": "https://registry.npmjs.org/safer-buffer/-/safer-buffer-2.1.2.tgz", "integrity": "sha512-YZo3K82SD7Riyi0E1EQPojLz7kpepnSQI9IyPbHHg1XXXevb5dJI7tpyN2ADxGcQbHG7vcyRHk0cbwqcQriUtg==" }, - "samsam": { - "version": "1.3.0", - "resolved": "https://registry.npmjs.org/samsam/-/samsam-1.3.0.tgz", - "integrity": "sha512-1HwIYD/8UlOtFS3QO3w7ey+SdSDFE4HRNLZoZRYVQefrOY3l17epswImeB1ijgJFQJodIaHcwkp3r/myBjFVbg==", - "dev": true - }, "sandboxed-module": { "version": "0.3.0", "resolved": "https://registry.npmjs.org/sandboxed-module/-/sandboxed-module-0.3.0.tgz", - "integrity": "sha512-/2IfB1wtca3eNVPXbQrb6UkhE/1pV4Wz+5CdG6DPYqeaDsYDzxglBT7/cVaqyrlRyQKdmw+uTZUTRos9FFD2PQ==", + "integrity": "sha1-8fvvvYCaT2kHO9B8rm/H2y6vX2o=", "dev": true, "requires": { "require-like": "0.1.2", @@ -5325,7 +5773,7 @@ "stack-trace": { "version": "0.0.6", "resolved": "https://registry.npmjs.org/stack-trace/-/stack-trace-0.0.6.tgz", - "integrity": "sha512-5/6uZt7RYjjAl8z2j1mXWAewz+I4Hk2/L/3n6NRLIQ31+uQ7nMd9O6G69QCdrrufHv0QGRRHl/jwUEGTqhelTA==", + "integrity": "sha1-HnGb1qJin/CcGJ4Xqe+QKpT8XbA=", "dev": true } } @@ -5367,6 +5815,15 @@ } } }, + "serialize-javascript": { + "version": "5.0.1", + "resolved": "https://registry.npmjs.org/serialize-javascript/-/serialize-javascript-5.0.1.tgz", + "integrity": "sha512-SaaNal9imEO737H2c05Og0/8LUXG7EnsZyMa8MzkmuHoELfT6txuj0cMqRj6zfPKnmQ1yasR4PCJc8x+M4JSPA==", + "dev": true, + "requires": { + "randombytes": "^2.1.0" + } + }, "serve-static": { "version": "1.14.1", "resolved": "https://registry.npmjs.org/serve-static/-/serve-static-1.14.1.tgz", @@ -5399,7 +5856,7 @@ "coffee-script": { "version": "1.6.0", "resolved": "https://registry.npmjs.org/coffee-script/-/coffee-script-1.6.0.tgz", - "integrity": "sha512-Tx8itEfCsQp8RbLDFt7qwjqXycAx2g6SI7//4PPUR2j6meLmNifYm6zKrNDcU1+Q/GWRhjhEZk7DaLG1TfIzGA==" + "integrity": "sha1-gIs5bhEPU9AhoZpO8fZb4OjjX6M=" } } }, @@ -5429,41 +5886,33 @@ "integrity": "sha512-VUJ49FC8U1OxwZLxIbTTrDvLnf/6TDgxZcK8wxR8zs13xpx7xbG60ndBlhNrFi2EMuFRoeDoJO7wthSLq42EjA==" }, "sinon": { - "version": "2.4.1", - "resolved": "https://registry.npmjs.org/sinon/-/sinon-2.4.1.tgz", - "integrity": "sha512-vFTrO9Wt0ECffDYIPSP/E5bBugt0UjcBQOfQUMh66xzkyPEnhl/vM2LRZi2ajuTdkH07sA6DzrM6KvdvGIH8xw==", + "version": "9.2.4", + "resolved": "https://registry.npmjs.org/sinon/-/sinon-9.2.4.tgz", + "integrity": "sha512-zljcULZQsJxVra28qIAL6ow1Z9tpattkCTEJR4RBP3TGc00FcttsP5pK284Nas5WjMZU5Yzy3kAIp3B3KRf5Yg==", "dev": true, "requires": { - "diff": "^3.1.0", - "formatio": "1.2.0", - "lolex": "^1.6.0", - "native-promise-only": "^0.8.1", - "path-to-regexp": "^1.7.0", - "samsam": "^1.1.3", - "text-encoding": "0.6.4", - "type-detect": "^4.0.0" + "@sinonjs/commons": "^1.8.1", + "@sinonjs/fake-timers": "^6.0.1", + "@sinonjs/samsam": "^5.3.1", + "diff": "^4.0.2", + "nise": "^4.0.4", + "supports-color": "^7.1.0" }, "dependencies": { - "isarray": { - "version": "0.0.1", - "resolved": "https://registry.npmjs.org/isarray/-/isarray-0.0.1.tgz", - "integrity": "sha512-D2S+3GLxWH+uhrNEcoh/fnmYeP8E8/zHl644d/jdA0g2uyXvy3sb0qxotE+ne0LtccHknQzWwZEzhak7oJ0COQ==", + "has-flag": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/has-flag/-/has-flag-4.0.0.tgz", + "integrity": "sha512-EykJT/Q1KjTWctppgIAgfSO0tKVuZUjhgMr17kqTumMl6Afv3EISleU7qZUzoXDFTAHTDC4NOoG/ZxU3EvlMPQ==", "dev": true }, - "path-to-regexp": { - "version": "1.8.0", - "resolved": "https://registry.npmjs.org/path-to-regexp/-/path-to-regexp-1.8.0.tgz", - "integrity": "sha512-n43JRhlUKUAlibEJhPeir1ncUID16QnEjNpwzNdO3Lm4ywrBpBZ5oLD0I6br9evr1Y9JTqwRtAh7JLoOzAQdVA==", + "supports-color": { + "version": "7.2.0", + "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-7.2.0.tgz", + "integrity": "sha512-qpCAvRl9stuOHveKsn7HncJRvv501qIacKzQlO/+Lwxc9+0q2wLyv4Dfvt80/DPn2pqOBsJdDiogXGR9+OvwRw==", "dev": true, "requires": { - "isarray": "0.0.1" + "has-flag": "^4.0.0" } - }, - "type-detect": { - "version": "4.0.8", - "resolved": "https://registry.npmjs.org/type-detect/-/type-detect-4.0.8.tgz", - "integrity": "sha512-0fr/mIH1dlO+x7TlcMy+bIDqKPsw/70tVyeHW787goQjhmqaZe10uwLujubK9q9Lg6Fiho1KUKDYz0Z7k7g5/g==", - "dev": true } } }, @@ -5607,12 +6056,12 @@ "standard-as-callback": { "version": "2.0.1", "resolved": "https://registry.npmjs.org/standard-as-callback/-/standard-as-callback-2.0.1.tgz", - "integrity": "sha512-NQOxSeB8gOI5WjSaxjBgog2QFw55FV8TkS6Y07BiB3VJ8xNTvUYm0wl0s8ObgQ5NhdpnNfigMIKjgPESzgr4tg==" + "integrity": "sha1-7YuyVkjhWDF1m2Ajvbh+a2CzgSY=" }, "statuses": { "version": "1.5.0", "resolved": "https://registry.npmjs.org/statuses/-/statuses-1.5.0.tgz", - "integrity": "sha512-OpZ3zP+jT1PI7I8nemJX4AKmAX070ZkYPVWV/AaKTJl+tXCTGyVdC1a4SL8RUQYEwk/f34ZX8UTykN68FwrqAA==" + "integrity": "sha1-Fhx9rBd2Wf2YEfQ3cfqZOBR4Yow=" }, "stream-events": { "version": "1.0.5", @@ -5786,12 +6235,6 @@ "bintrees": "1.0.1" } }, - "text-encoding": { - "version": "0.6.4", - "resolved": "https://registry.npmjs.org/text-encoding/-/text-encoding-0.6.4.tgz", - "integrity": "sha512-hJnc6Qg3dWoOMkqP53F0dzRIgtmsAge09kxUIqGrEUS4qr5rWLckGYaQAVr+opBrIMRErGgy6f5aPnyPpyGRfg==", - "dev": true - }, "text-table": { "version": "0.2.0", "resolved": "https://registry.npmjs.org/text-table/-/text-table-0.2.0.tgz", @@ -5801,7 +6244,7 @@ "through": { "version": "2.3.8", "resolved": "https://registry.npmjs.org/through/-/through-2.3.8.tgz", - "integrity": "sha512-w89qg7PI8wAdvX60bMDP+bFoD5Dvhm9oLheFp5O4a2QF0cSBGsBX4qZmadPMvVqlLJBBci+WqGGOAPvcDeNSVg==" + "integrity": "sha1-DdTJ/6q8NXlgsbckEV1+Doai4fU=" }, "through2": { "version": "3.0.1", @@ -5819,7 +6262,7 @@ "timekeeper": { "version": "0.0.4", "resolved": "https://registry.npmjs.org/timekeeper/-/timekeeper-0.0.4.tgz", - "integrity": "sha512-QSNovcsPIbrI9zzXTesL/iiDrS+4IT+0xCxFzDSI2/yHkHVL1QEB5FhzrKMzCEwsbSOISEsH1yPUZ6/Fve9DZQ==", + "integrity": "sha1-kNt58X2Ni1NiFUOJSSuXJ2LP0nY=", "dev": true }, "tinycolor": { @@ -5841,6 +6284,15 @@ "resolved": "https://registry.npmjs.org/to-no-case/-/to-no-case-1.0.2.tgz", "integrity": "sha1-xyKQcWTvaxeBMsjmmTAhLRtKoWo=" }, + "to-regex-range": { + "version": "5.0.1", + "resolved": "https://registry.npmjs.org/to-regex-range/-/to-regex-range-5.0.1.tgz", + "integrity": "sha512-65P7iz6X5yEr1cwcgvQxbbIw7Uk3gOy5dIdtZ4rDveLqhrdJP+Li/Hx6tyK0NEb+2GCyneCMJiGqrADCSNk8sQ==", + "dev": true, + "requires": { + "is-number": "^7.0.0" + } + }, "to-snake-case": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/to-snake-case/-/to-snake-case-1.0.0.tgz", @@ -5891,7 +6343,7 @@ "tunnel-agent": { "version": "0.6.0", "resolved": "https://registry.npmjs.org/tunnel-agent/-/tunnel-agent-0.6.0.tgz", - "integrity": "sha512-McnNiV1l8RYeY8tBgEpuodCC1mLUdbSN+CYBL7kJsJNInOP8UjDDEwdk6Mw60vdLLrr5NHKZhMAOSrR2NZuQ+w==", + "integrity": "sha1-J6XeoGs2sEoKmWZ3SykIaPD8QP0=", "requires": { "safe-buffer": "^5.0.1" } @@ -5899,7 +6351,7 @@ "tweetnacl": { "version": "0.14.5", "resolved": "https://registry.npmjs.org/tweetnacl/-/tweetnacl-0.14.5.tgz", - "integrity": "sha512-KXXFFdAbFXY4geFIwoyNK+f5Z1b7swfXABfL7HXCmoIWMKU3dmS26672A4EeQtDzLKy7SXmfBu51JolvEKwtGA==" + "integrity": "sha1-WuaBd/GS1EViadEIr6k/+HQ/T2Q=" }, "type-check": { "version": "0.3.2", @@ -5913,7 +6365,7 @@ "type-detect": { "version": "0.1.1", "resolved": "https://registry.npmjs.org/type-detect/-/type-detect-0.1.1.tgz", - "integrity": "sha512-5rqszGVwYgBoDkIm2oUtvkfZMQ0vk29iDMU0W2qCa3rG0vPDNczCMT4hV/bLBgLg8k8ri6+u3Zbt+S/14eMzlA==", + "integrity": "sha1-C6XsKohWQORw6k6FBZcZANrFiCI=", "dev": true }, "type-fest": { @@ -5973,7 +6425,7 @@ "unpipe": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/unpipe/-/unpipe-1.0.0.tgz", - "integrity": "sha512-pjy2bYhSsufwWlKwPc+l3cN7+wuJlK6uz0YdJEOlQDbl6jo/YlPi4mb8agUkVC8BF7V8NuzeyPNqRksA3hztKQ==" + "integrity": "sha1-sr9O6FFKrmFltIF4KdIbLvSZBOw=" }, "uri-js": { "version": "4.2.2", @@ -5986,12 +6438,12 @@ "util-deprecate": { "version": "1.0.2", "resolved": "https://registry.npmjs.org/util-deprecate/-/util-deprecate-1.0.2.tgz", - "integrity": "sha512-EPD5q1uXyFxJpCrLnCc1nHnq3gOa6DZBocAIiI2TaSCA7VCJ1UJDMagCzIkXNsUYfD1daK//LTEQ8xiIbrHtcw==" + "integrity": "sha1-RQ1Nyfpw3nMnYvvS1KKJgUGaDM8=" }, "utils-merge": { "version": "1.0.1", "resolved": "https://registry.npmjs.org/utils-merge/-/utils-merge-1.0.1.tgz", - "integrity": "sha512-pMZTvIkT1d+TFGvDOqodOclx0QWkkgi6Tdoa8gC8ffGAAqz9pzPTZWAybbsHHoED/ztMtkv/VoYTYyShUn81hA==" + "integrity": "sha1-n5VxD1CiZ5R7LMwSR0HBAoQn5xM=" }, "uuid": { "version": "7.0.3", @@ -6017,12 +6469,12 @@ "vary": { "version": "1.1.2", "resolved": "https://registry.npmjs.org/vary/-/vary-1.1.2.tgz", - "integrity": "sha512-BNGbWLfd0eUPabhkXUVm0j8uuvREyTh5ovRa/dyow/BqAbZJyC+5fU+IzQOzmAKzYqYRAISoRhdQr3eIZ/PXqg==" + "integrity": "sha1-IpnwLG3tMNSllhsLn3RSShj2NPw=" }, "verror": { "version": "1.10.0", "resolved": "https://registry.npmjs.org/verror/-/verror-1.10.0.tgz", - "integrity": "sha512-ZZKSmDAEFOijERBLkmYfJ+vmk3w+7hOLYDNkRCuRuMJGEmqYNCNLyBBFwWKVMhfwaEF3WOd0Zlw86U/WC/+nYw==", + "integrity": "sha1-OhBcoXBTr1XW4nDB+CiGguGNpAA=", "requires": { "assert-plus": "^1.0.0", "core-util-is": "1.0.2", @@ -6166,6 +6618,12 @@ "integrity": "sha512-Hz/mrNwitNRh/HUAtM/VT/5VH+ygD6DV7mYKZAtHOrbs8U7lvPS6xf7EJKMF0uW1KJCl0H701g3ZGus+muE5vQ==", "dev": true }, + "workerpool": { + "version": "6.1.0", + "resolved": "https://registry.npmjs.org/workerpool/-/workerpool-6.1.0.tgz", + "integrity": "sha512-toV7q9rWNYha963Pl/qyeZ6wG+3nnsyvolaNUS8+R5Wtw6qJPTxIlOP1ZSvcGhEJw+l3HMMmtiNo9Gl61G4GVg==", + "dev": true + }, "wrap-ansi": { "version": "5.1.0", "resolved": "https://registry.npmjs.org/wrap-ansi/-/wrap-ansi-5.1.0.tgz", @@ -6205,7 +6663,7 @@ "wrappy": { "version": "1.0.2", "resolved": "https://registry.npmjs.org/wrappy/-/wrappy-1.0.2.tgz", - "integrity": "sha512-l4Sp/DRseor9wL6EvV2+TuQn63dMkPjZ/sp9XkghTEbV9KlPS1xUsZ3u7/IQO4wxtcFB4bgpQPRcR3QCvezPcQ==" + "integrity": "sha1-tSQ9jz7BqjXxNkYFvA0QNuMKtp8=" }, "write": { "version": "1.0.3", @@ -6331,6 +6789,32 @@ "decamelize": "^1.2.0" } }, + "yargs-unparser": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/yargs-unparser/-/yargs-unparser-2.0.0.tgz", + "integrity": "sha512-7pRTIA9Qc1caZ0bZ6RYRGbHJthJWuakf+WmHK0rVeLkNrrGhfoabBNdue6kdINI6r4if7ocq9aD/n7xwKOdzOA==", + "dev": true, + "requires": { + "camelcase": "^6.0.0", + "decamelize": "^4.0.0", + "flat": "^5.0.2", + "is-plain-obj": "^2.1.0" + }, + "dependencies": { + "camelcase": { + "version": "6.2.0", + "resolved": "https://registry.npmjs.org/camelcase/-/camelcase-6.2.0.tgz", + "integrity": "sha512-c7wVvbw3f37nuobQNtgsgG9POC9qMbNuMQmTCqZv23b6MIz0fcYpBiOlv9gEN/hdLdnZTDQhg6e9Dq5M1vKvfg==", + "dev": true + }, + "decamelize": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/decamelize/-/decamelize-4.0.0.tgz", + "integrity": "sha512-9iE1PgSik9HeIIw2JO94IidnE3eBoQrFJ3w7sFuzSX4DpmZ3v5sZpUiV5Swcf6mQEF+Y0ru8Neo+p+nyh2J+hQ==", + "dev": true + } + } + }, "yn": { "version": "3.1.1", "resolved": "https://registry.npmjs.org/yn/-/yn-3.1.1.tgz", diff --git a/services/real-time/package.json b/services/real-time/package.json index 3a2145a85e..51ee7a1a0e 100644 --- a/services/real-time/package.json +++ b/services/real-time/package.json @@ -54,11 +54,11 @@ "eslint-plugin-prettier": "^3.1.2", "eslint-plugin-promise": "^4.2.1", "eslint-plugin-standard": "^4.0.1", - "mocha": "^4.0.1", + "mocha": "^8.3.2", "prettier": "^2.0.0", "prettier-eslint-cli": "^5.0.0", "sandboxed-module": "~0.3.0", - "sinon": "^2.4.1", + "sinon": "^9.2.4", "timekeeper": "0.0.4", "uid-safe": "^2.1.5" } diff --git a/services/real-time/test/acceptance/js/ApplyUpdateTests.js b/services/real-time/test/acceptance/js/ApplyUpdateTests.js index 27f77b973f..14a8faa186 100644 --- a/services/real-time/test/acceptance/js/ApplyUpdateTests.js +++ b/services/real-time/test/acceptance/js/ApplyUpdateTests.js @@ -13,9 +13,7 @@ * Full docs: https://github.com/decaffeinate/decaffeinate/blob/master/docs/suggestions.md */ const async = require('async') -const chai = require('chai') -const { expect } = chai -chai.should() +const { expect } = require('chai') const RealTimeClient = require('./helpers/RealTimeClient') const FixturesManager = require('./helpers/FixturesManager') diff --git a/services/real-time/test/acceptance/js/ClientTrackingTests.js b/services/real-time/test/acceptance/js/ClientTrackingTests.js index 079baadb58..702b2288ab 100644 --- a/services/real-time/test/acceptance/js/ClientTrackingTests.js +++ b/services/real-time/test/acceptance/js/ClientTrackingTests.js @@ -12,9 +12,7 @@ * DS207: Consider shorter variations of null checks * Full docs: https://github.com/decaffeinate/decaffeinate/blob/master/docs/suggestions.md */ -const chai = require('chai') -const { expect } = chai -chai.should() +const { expect } = require('chai') const RealTimeClient = require('./helpers/RealTimeClient') const MockWebServer = require('./helpers/MockWebServer') diff --git a/services/real-time/test/acceptance/js/JoinDocTests.js b/services/real-time/test/acceptance/js/JoinDocTests.js index 217731a1df..936b8c8e04 100644 --- a/services/real-time/test/acceptance/js/JoinDocTests.js +++ b/services/real-time/test/acceptance/js/JoinDocTests.js @@ -11,9 +11,7 @@ * DS102: Remove unnecessary code created because of implicit returns * Full docs: https://github.com/decaffeinate/decaffeinate/blob/master/docs/suggestions.md */ -const chai = require('chai') -const { expect } = chai -chai.should() +const { expect } = require('chai') const RealTimeClient = require('./helpers/RealTimeClient') const MockDocUpdaterServer = require('./helpers/MockDocUpdaterServer') diff --git a/services/real-time/test/acceptance/js/JoinProjectTests.js b/services/real-time/test/acceptance/js/JoinProjectTests.js index 508e2b6614..61ede26f87 100644 --- a/services/real-time/test/acceptance/js/JoinProjectTests.js +++ b/services/real-time/test/acceptance/js/JoinProjectTests.js @@ -10,9 +10,7 @@ * DS102: Remove unnecessary code created because of implicit returns * Full docs: https://github.com/decaffeinate/decaffeinate/blob/master/docs/suggestions.md */ -const chai = require('chai') -const { expect } = chai -chai.should() +const { expect } = require('chai') const RealTimeClient = require('./helpers/RealTimeClient') const MockWebServer = require('./helpers/MockWebServer') diff --git a/services/real-time/test/acceptance/js/LeaveDocTests.js b/services/real-time/test/acceptance/js/LeaveDocTests.js index a842087522..fc77da6314 100644 --- a/services/real-time/test/acceptance/js/LeaveDocTests.js +++ b/services/real-time/test/acceptance/js/LeaveDocTests.js @@ -13,9 +13,7 @@ * DS207: Consider shorter variations of null checks * Full docs: https://github.com/decaffeinate/decaffeinate/blob/master/docs/suggestions.md */ -const chai = require('chai') -const { expect } = chai -chai.should() +const { expect } = require('chai') const sinon = require('sinon') const RealTimeClient = require('./helpers/RealTimeClient') diff --git a/services/real-time/test/acceptance/js/MatrixTests.js b/services/real-time/test/acceptance/js/MatrixTests.js index 2f4b362bca..9da735083e 100644 --- a/services/real-time/test/acceptance/js/MatrixTests.js +++ b/services/real-time/test/acceptance/js/MatrixTests.js @@ -47,8 +47,7 @@ There is additional meta-data that UserItems and SessionItems may use to skip /* eslint-disable camelcase, */ -const chai = require('chai') -const { expect } = chai +const { expect } = require('chai') const async = require('async') const RealTimeClient = require('./helpers/RealTimeClient') diff --git a/services/real-time/test/acceptance/js/ReceiveUpdateTests.js b/services/real-time/test/acceptance/js/ReceiveUpdateTests.js index 75d3991fc3..ae7ab8ef55 100644 --- a/services/real-time/test/acceptance/js/ReceiveUpdateTests.js +++ b/services/real-time/test/acceptance/js/ReceiveUpdateTests.js @@ -11,9 +11,7 @@ * DS207: Consider shorter variations of null checks * Full docs: https://github.com/decaffeinate/decaffeinate/blob/master/docs/suggestions.md */ -const chai = require('chai') -const { expect } = chai -chai.should() +const { expect } = require('chai') const RealTimeClient = require('./helpers/RealTimeClient') const MockWebServer = require('./helpers/MockWebServer') diff --git a/services/real-time/test/acceptance/js/SessionTests.js b/services/real-time/test/acceptance/js/SessionTests.js index 941a59f4b9..858a2dd48a 100644 --- a/services/real-time/test/acceptance/js/SessionTests.js +++ b/services/real-time/test/acceptance/js/SessionTests.js @@ -11,8 +11,7 @@ * DS207: Consider shorter variations of null checks * Full docs: https://github.com/decaffeinate/decaffeinate/blob/master/docs/suggestions.md */ -const chai = require('chai') -const { expect } = chai +const { expect } = require('chai') const RealTimeClient = require('./helpers/RealTimeClient') diff --git a/services/real-time/test/setup.js b/services/real-time/test/setup.js new file mode 100644 index 0000000000..90f4363c52 --- /dev/null +++ b/services/real-time/test/setup.js @@ -0,0 +1,38 @@ +const chai = require('chai') +const SandboxedModule = require('sandboxed-module') +const sinon = require('sinon') + +// Chai configuration +chai.should() + +// Global stubs +const sandbox = sinon.createSandbox() +const stubs = { + logger: { + debug: sandbox.stub(), + log: sandbox.stub(), + info: sandbox.stub(), + warn: sandbox.stub(), + err: sandbox.stub(), + error: sandbox.stub() + } +} + +// SandboxedModule configuration +SandboxedModule.configure({ + requires: { + 'logger-sharelatex': stubs.logger + }, + globals: { Buffer, JSON, console, process } +}) + +// Mocha hooks +exports.mochaHooks = { + beforeEach() { + this.logger = stubs.logger + }, + + afterEach() { + sandbox.reset() + } +} diff --git a/services/real-time/test/unit/js/AuthorizationManagerTests.js b/services/real-time/test/unit/js/AuthorizationManagerTests.js index 3093017a39..d8a3f7ae66 100644 --- a/services/real-time/test/unit/js/AuthorizationManagerTests.js +++ b/services/real-time/test/unit/js/AuthorizationManagerTests.js @@ -9,9 +9,7 @@ * DS102: Remove unnecessary code created because of implicit returns * Full docs: https://github.com/decaffeinate/decaffeinate/blob/master/docs/suggestions.md */ -const chai = require('chai') -chai.should() -const { expect } = chai +const { expect } = require('chai') const sinon = require('sinon') const SandboxedModule = require('sandboxed-module') const path = require('path') diff --git a/services/real-time/test/unit/js/ChannelManagerTests.js b/services/real-time/test/unit/js/ChannelManagerTests.js index f76a109129..aad5ddbd42 100644 --- a/services/real-time/test/unit/js/ChannelManagerTests.js +++ b/services/real-time/test/unit/js/ChannelManagerTests.js @@ -9,9 +9,7 @@ * DS102: Remove unnecessary code created because of implicit returns * Full docs: https://github.com/decaffeinate/decaffeinate/blob/master/docs/suggestions.md */ -const chai = require('chai') -const should = chai.should() -const { expect } = chai +const { expect } = require('chai') const sinon = require('sinon') const modulePath = '../../../app/js/ChannelManager.js' const SandboxedModule = require('sandboxed-module') @@ -26,11 +24,6 @@ describe('ChannelManager', function () { '@overleaf/metrics': (this.metrics = { inc: sinon.stub(), summary: sinon.stub() - }), - 'logger-sharelatex': (this.logger = { - log: sinon.stub(), - warn: sinon.stub(), - error: sinon.stub() }) } })) diff --git a/services/real-time/test/unit/js/ConnectedUsersManagerTests.js b/services/real-time/test/unit/js/ConnectedUsersManagerTests.js index 6489b20639..a0ab01631e 100644 --- a/services/real-time/test/unit/js/ConnectedUsersManagerTests.js +++ b/services/real-time/test/unit/js/ConnectedUsersManagerTests.js @@ -12,7 +12,6 @@ * Full docs: https://github.com/decaffeinate/decaffeinate/blob/master/docs/suggestions.md */ -const should = require('chai').should() const SandboxedModule = require('sandboxed-module') const assert = require('assert') const path = require('path') @@ -58,7 +57,6 @@ describe('ConnectedUsersManager', function () { this.ConnectedUsersManager = SandboxedModule.require(modulePath, { requires: { 'settings-sharelatex': this.settings, - 'logger-sharelatex': { log() {} }, '@overleaf/redis-wrapper': { createClient: () => { return this.rClient diff --git a/services/real-time/test/unit/js/DocumentUpdaterControllerTests.js b/services/real-time/test/unit/js/DocumentUpdaterControllerTests.js index 0aabc51de7..8f0be7f87d 100644 --- a/services/real-time/test/unit/js/DocumentUpdaterControllerTests.js +++ b/services/real-time/test/unit/js/DocumentUpdaterControllerTests.js @@ -12,7 +12,6 @@ */ const SandboxedModule = require('sandboxed-module') const sinon = require('sinon') -require('chai').should() const modulePath = require('path').join( __dirname, '../../../app/js/DocumentUpdaterController' @@ -29,11 +28,6 @@ describe('DocumentUpdaterController', function () { this.RoomEvents = { on: sinon.stub() } this.EditorUpdatesController = SandboxedModule.require(modulePath, { requires: { - 'logger-sharelatex': (this.logger = { - error: sinon.stub(), - log: sinon.stub(), - warn: sinon.stub() - }), 'settings-sharelatex': (this.settings = { redis: { documentupdater: { diff --git a/services/real-time/test/unit/js/DocumentUpdaterManagerTests.js b/services/real-time/test/unit/js/DocumentUpdaterManagerTests.js index 2a5fdf530f..94ef11f716 100644 --- a/services/real-time/test/unit/js/DocumentUpdaterManagerTests.js +++ b/services/real-time/test/unit/js/DocumentUpdaterManagerTests.js @@ -10,7 +10,6 @@ * DS102: Remove unnecessary code created because of implicit returns * Full docs: https://github.com/decaffeinate/decaffeinate/blob/master/docs/suggestions.md */ -require('chai').should() const sinon = require('sinon') const SandboxedModule = require('sandboxed-module') const path = require('path') @@ -43,11 +42,6 @@ describe('DocumentUpdaterManager', function () { return (this.DocumentUpdaterManager = SandboxedModule.require(modulePath, { requires: { 'settings-sharelatex': this.settings, - 'logger-sharelatex': (this.logger = { - log: sinon.stub(), - error: sinon.stub(), - warn: sinon.stub() - }), request: (this.request = {}), '@overleaf/redis-wrapper': { createClient: () => this.rclient }, '@overleaf/metrics': (this.Metrics = { @@ -56,9 +50,6 @@ describe('DocumentUpdaterManager', function () { done() {} }) }) - }, - globals: { - JSON: (this.JSON = Object.create(JSON)) } })) }) // avoid modifying JSON object directly @@ -325,7 +316,9 @@ describe('DocumentUpdaterManager', function () { describe('with null byte corruption', function () { beforeEach(function () { - this.JSON.stringify = () => '["bad bytes! \u0000 <- here"]' + this.stringifyStub = sinon + .stub(JSON, 'stringify') + .callsFake(() => '["bad bytes! \u0000 <- here"]') return this.DocumentUpdaterManager.queueChange( this.project_id, this.doc_id, @@ -334,6 +327,10 @@ describe('DocumentUpdaterManager', function () { ) }) + afterEach(function () { + this.stringifyStub.restore() + }) + it('should return an error', function () { return this.callback .calledWithExactly(sinon.match(Error)) diff --git a/services/real-time/test/unit/js/DrainManagerTests.js b/services/real-time/test/unit/js/DrainManagerTests.js index 7ed7f1e06e..bf8de6a6e9 100644 --- a/services/real-time/test/unit/js/DrainManagerTests.js +++ b/services/real-time/test/unit/js/DrainManagerTests.js @@ -9,7 +9,6 @@ * DS102: Remove unnecessary code created because of implicit returns * Full docs: https://github.com/decaffeinate/decaffeinate/blob/master/docs/suggestions.md */ -const should = require('chai').should() const sinon = require('sinon') const SandboxedModule = require('sandboxed-module') const path = require('path') @@ -17,11 +16,7 @@ const modulePath = path.join(__dirname, '../../../app/js/DrainManager') describe('DrainManager', function () { beforeEach(function () { - this.DrainManager = SandboxedModule.require(modulePath, { - requires: { - 'logger-sharelatex': (this.logger = { log: sinon.stub() }) - } - }) + this.DrainManager = SandboxedModule.require(modulePath, {}) return (this.io = { sockets: { clients: sinon.stub() diff --git a/services/real-time/test/unit/js/EventLoggerTests.js b/services/real-time/test/unit/js/EventLoggerTests.js index c95d65ae2e..340e17184e 100644 --- a/services/real-time/test/unit/js/EventLoggerTests.js +++ b/services/real-time/test/unit/js/EventLoggerTests.js @@ -8,7 +8,6 @@ * DS102: Remove unnecessary code created because of implicit returns * Full docs: https://github.com/decaffeinate/decaffeinate/blob/master/docs/suggestions.md */ -require('chai').should() const { expect } = require('chai') const SandboxedModule = require('sandboxed-module') const modulePath = '../../../app/js/EventLogger' @@ -21,10 +20,6 @@ describe('EventLogger', function () { tk.freeze(new Date(this.start)) this.EventLogger = SandboxedModule.require(modulePath, { requires: { - 'logger-sharelatex': (this.logger = { - error: sinon.stub(), - warn: sinon.stub() - }), '@overleaf/metrics': (this.metrics = { inc: sinon.stub() }) } }) diff --git a/services/real-time/test/unit/js/RoomManagerTests.js b/services/real-time/test/unit/js/RoomManagerTests.js index 84aeb7a868..fd22271469 100644 --- a/services/real-time/test/unit/js/RoomManagerTests.js +++ b/services/real-time/test/unit/js/RoomManagerTests.js @@ -10,9 +10,7 @@ * DS102: Remove unnecessary code created because of implicit returns * Full docs: https://github.com/decaffeinate/decaffeinate/blob/master/docs/suggestions.md */ -const chai = require('chai') -const { expect } = chai -const should = chai.should() +const { expect } = require('chai') const sinon = require('sinon') const modulePath = '../../../app/js/RoomManager.js' const SandboxedModule = require('sandboxed-module') @@ -26,11 +24,6 @@ describe('RoomManager', function () { this.RoomManager = SandboxedModule.require(modulePath, { requires: { 'settings-sharelatex': (this.settings = {}), - 'logger-sharelatex': (this.logger = { - log: sinon.stub(), - warn: sinon.stub(), - error: sinon.stub() - }), '@overleaf/metrics': (this.metrics = { gauge: sinon.stub() }) } }) diff --git a/services/real-time/test/unit/js/SafeJsonParseTest.js b/services/real-time/test/unit/js/SafeJsonParseTest.js index 51a60dea23..de50bccebb 100644 --- a/services/real-time/test/unit/js/SafeJsonParseTest.js +++ b/services/real-time/test/unit/js/SafeJsonParseTest.js @@ -11,11 +11,9 @@ * DS102: Remove unnecessary code created because of implicit returns * Full docs: https://github.com/decaffeinate/decaffeinate/blob/master/docs/suggestions.md */ -require('chai').should() const { expect } = require('chai') const SandboxedModule = require('sandboxed-module') const modulePath = '../../../app/js/SafeJsonParse' -const sinon = require('sinon') describe('SafeJsonParse', function () { beforeEach(function () { @@ -23,8 +21,7 @@ describe('SafeJsonParse', function () { requires: { 'settings-sharelatex': (this.Settings = { maxUpdateSize: 16 * 1024 - }), - 'logger-sharelatex': (this.logger = { error: sinon.stub() }) + }) } })) }) diff --git a/services/real-time/test/unit/js/WebApiManagerTests.js b/services/real-time/test/unit/js/WebApiManagerTests.js index a366a2e75d..97be3c27f0 100644 --- a/services/real-time/test/unit/js/WebApiManagerTests.js +++ b/services/real-time/test/unit/js/WebApiManagerTests.js @@ -9,8 +9,6 @@ * DS102: Remove unnecessary code created because of implicit returns * Full docs: https://github.com/decaffeinate/decaffeinate/blob/master/docs/suggestions.md */ -const chai = require('chai') -const should = chai.should() const sinon = require('sinon') const modulePath = '../../../app/js/WebApiManager.js' const SandboxedModule = require('sandboxed-module') @@ -33,10 +31,6 @@ describe('WebApiManager', function () { pass: 'password' } } - }), - 'logger-sharelatex': (this.logger = { - log: sinon.stub(), - error: sinon.stub() }) } })) diff --git a/services/real-time/test/unit/js/WebsocketControllerTests.js b/services/real-time/test/unit/js/WebsocketControllerTests.js index c5f4a1ac42..e227a1ce82 100644 --- a/services/real-time/test/unit/js/WebsocketControllerTests.js +++ b/services/real-time/test/unit/js/WebsocketControllerTests.js @@ -12,10 +12,8 @@ * DS102: Remove unnecessary code created because of implicit returns * Full docs: https://github.com/decaffeinate/decaffeinate/blob/master/docs/suggestions.md */ -const chai = require('chai') -const should = chai.should() const sinon = require('sinon') -const { expect } = chai +const { expect } = require('chai') const modulePath = '../../../app/js/WebsocketController.js' const SandboxedModule = require('sandboxed-module') const tk = require('timekeeper') @@ -50,11 +48,6 @@ describe('WebsocketController', function () { './DocumentUpdaterManager': (this.DocumentUpdaterManager = {}), './ConnectedUsersManager': (this.ConnectedUsersManager = {}), './WebsocketLoadBalancer': (this.WebsocketLoadBalancer = {}), - 'logger-sharelatex': (this.logger = { - log: sinon.stub(), - error: sinon.stub(), - warn: sinon.stub() - }), '@overleaf/metrics': (this.metrics = { inc: sinon.stub(), set: sinon.stub() diff --git a/services/real-time/test/unit/js/WebsocketLoadBalancerTests.js b/services/real-time/test/unit/js/WebsocketLoadBalancerTests.js index 355e635353..9e61501305 100644 --- a/services/real-time/test/unit/js/WebsocketLoadBalancerTests.js +++ b/services/real-time/test/unit/js/WebsocketLoadBalancerTests.js @@ -11,7 +11,6 @@ */ const SandboxedModule = require('sandboxed-module') const sinon = require('sinon') -require('chai').should() const modulePath = require('path').join( __dirname, '../../../app/js/WebsocketLoadBalancer' @@ -26,10 +25,6 @@ describe('WebsocketLoadBalancer', function () { './RedisClientManager': { createClientList: () => [] }, - 'logger-sharelatex': (this.logger = { - log: sinon.stub(), - error: sinon.stub() - }), './SafeJsonParse': (this.SafeJsonParse = { parse: (data, cb) => cb(null, JSON.parse(data)) }), From b27cb4ec128578ac0ff1900cbc8133bd75eae663 Mon Sep 17 00:00:00 2001 From: Eric Mc Sween Date: Thu, 18 Mar 2021 16:20:29 -0400 Subject: [PATCH 466/491] Upgrade to Node 12 --- services/real-time/.nvmrc | 2 +- services/real-time/Dockerfile | 2 +- services/real-time/Makefile | 6 ++++-- services/real-time/buildscript.txt | 4 ++-- services/real-time/docker-compose.yml | 4 ++-- 5 files changed, 10 insertions(+), 8 deletions(-) diff --git a/services/real-time/.nvmrc b/services/real-time/.nvmrc index 2baa2d433a..e68b860383 100644 --- a/services/real-time/.nvmrc +++ b/services/real-time/.nvmrc @@ -1 +1 @@ -10.23.1 +12.21.0 diff --git a/services/real-time/Dockerfile b/services/real-time/Dockerfile index 2da67d2436..4f417a2a4b 100644 --- a/services/real-time/Dockerfile +++ b/services/real-time/Dockerfile @@ -2,7 +2,7 @@ # Instead run bin/update_build_scripts from # https://github.com/sharelatex/sharelatex-dev-environment -FROM node:10.23.1 as base +FROM node:12.21.0 as base WORKDIR /app diff --git a/services/real-time/Makefile b/services/real-time/Makefile index a569868e5b..9e065f2220 100644 --- a/services/real-time/Makefile +++ b/services/real-time/Makefile @@ -21,8 +21,10 @@ DOCKER_COMPOSE_TEST_UNIT = \ COMPOSE_PROJECT_NAME=test_unit_$(BUILD_DIR_NAME) $(DOCKER_COMPOSE) clean: - docker rmi ci/$(PROJECT_NAME):$(BRANCH_NAME)-$(BUILD_NUMBER) - docker rmi gcr.io/overleaf-ops/$(PROJECT_NAME):$(BRANCH_NAME)-$(BUILD_NUMBER) + -docker rmi ci/$(PROJECT_NAME):$(BRANCH_NAME)-$(BUILD_NUMBER) + -docker rmi gcr.io/overleaf-ops/$(PROJECT_NAME):$(BRANCH_NAME)-$(BUILD_NUMBER) + -$(DOCKER_COMPOSE_TEST_UNIT) down --rmi local + -$(DOCKER_COMPOSE_TEST_ACCEPTANCE) down --rmi local format: $(DOCKER_COMPOSE) run --rm test_unit npm run --silent format diff --git a/services/real-time/buildscript.txt b/services/real-time/buildscript.txt index 6fbccbe4c4..109e42ca58 100644 --- a/services/real-time/buildscript.txt +++ b/services/real-time/buildscript.txt @@ -3,6 +3,6 @@ real-time --docker-repos=gcr.io/overleaf-ops --env-add= --env-pass-through= ---node-version=10.23.1 +--node-version=12.21.0 --public-repo=True ---script-version=3.4.0 +--script-version=3.7.0 diff --git a/services/real-time/docker-compose.yml b/services/real-time/docker-compose.yml index 8233095495..ad04d1ab54 100644 --- a/services/real-time/docker-compose.yml +++ b/services/real-time/docker-compose.yml @@ -6,7 +6,7 @@ version: "2.3" services: test_unit: - image: node:10.23.1 + image: node:12.21.0 volumes: - .:/app working_dir: /app @@ -18,7 +18,7 @@ services: user: node test_acceptance: - image: node:10.23.1 + image: node:12.21.0 volumes: - .:/app working_dir: /app From 17e30684993a46801895d074bfb13323578ab19b Mon Sep 17 00:00:00 2001 From: Brian Gough Date: Fri, 19 Mar 2021 16:27:27 +0000 Subject: [PATCH 467/491] close real-time via a status file --- services/real-time/app.js | 15 +++++- .../real-time/app/js/DeploymentManager.js | 53 +++++++++++++++++++ .../real-time/config/settings.defaults.js | 6 +++ 3 files changed, 72 insertions(+), 2 deletions(-) create mode 100644 services/real-time/app/js/DeploymentManager.js diff --git a/services/real-time/app.js b/services/real-time/app.js index e65561cf43..832bf52a53 100644 --- a/services/real-time/app.js +++ b/services/real-time/app.js @@ -22,6 +22,7 @@ const CookieParser = require('cookie-parser') const DrainManager = require('./app/js/DrainManager') const HealthCheckManager = require('./app/js/HealthCheckManager') +const DeploymentManager = require('./app/js/DeploymentManager') // NOTE: debug is invoked for every blob that is put on the wire const socketIoLogger = { @@ -36,6 +37,9 @@ const socketIoLogger = { log() {} } +// monitor status file to take dark deployments out of the load-balancer +DeploymentManager.initialise() + // Set up socket.io server const app = express() @@ -79,13 +83,20 @@ io.configure(function () { }) // a 200 response on '/' is required for load balancer health checks -app.get('/', (req, res) => res.send('real-time-sharelatex is alive')) +// these operate separately from kubernetes readiness checks +app.get('/', function (req, res) { + if (Settings.serviceIsClosed || Settings.shutDownInProgress) { + res.sendStatus(503) // Service unavailable + } else { + res.send('real-time is open') + } +}) app.get('/status', function (req, res) { if (Settings.shutDownInProgress) { res.sendStatus(503) // Service unavailable } else { - res.send('real-time-sharelatex is alive') + res.send('real-time is alive') } }) diff --git a/services/real-time/app/js/DeploymentManager.js b/services/real-time/app/js/DeploymentManager.js new file mode 100644 index 0000000000..a86db02dd1 --- /dev/null +++ b/services/real-time/app/js/DeploymentManager.js @@ -0,0 +1,53 @@ +const logger = require('logger-sharelatex') +const settings = require('settings-sharelatex') +const fs = require('fs') + +// Monitor a status file (e.g. /etc/real_time_status) periodically and close the +// service if the file contents don't contain the matching deployment colour. + +const FILE_CHECK_INTERVAL = 5000 +const statusFile = settings.deploymentFile +const deploymentColour = settings.deploymentColour + +function updateDeploymentStatus(fileContent) { + const closed = fileContent && fileContent.indexOf(deploymentColour) === -1 + if (closed && !settings.serviceIsClosed) { + settings.serviceIsClosed = true + logger.warn({ fileContent }, 'closing service') + } else if (!closed && settings.serviceIsClosed) { + settings.serviceIsClosed = false + logger.warn({ fileContent }, 'opening service') + } +} + +function pollStatusFile() { + fs.readFile(statusFile, { encoding: 'utf8' }, (err, fileContent) => { + if (err) { + logger.error( + { file: statusFile, fsErr: err }, + 'error reading service status file' + ) + return + } + updateDeploymentStatus(fileContent) + }) +} + +function checkStatusFileSync() { + // crash on start up if file does not exist + const content = fs.readFileSync(statusFile, { encoding: 'utf8' }) + updateDeploymentStatus(content) +} + +module.exports = { + initialise() { + if (statusFile && deploymentColour) { + logger.log( + { statusFile, deploymentColour, interval: FILE_CHECK_INTERVAL }, + 'monitoring deployment status file' + ) + checkStatusFileSync() // perform an initial synchronous check at start up + setInterval(pollStatusFile, FILE_CHECK_INTERVAL) // continue checking periodically + } + } +} diff --git a/services/real-time/config/settings.defaults.js b/services/real-time/config/settings.defaults.js index 486b686083..c15ca57f6f 100644 --- a/services/real-time/config/settings.defaults.js +++ b/services/real-time/config/settings.defaults.js @@ -148,6 +148,12 @@ const settings = { statusCheckInterval: parseInt(process.env.STATUS_CHECK_INTERVAL || '0'), + // The deployment colour for this app (if any). Used for blue green deploys. + deploymentColour: process.env.DEPLOYMENT_COLOUR, + // Load balancer health checks will return 200 only when this file contains + // the deployment colour for this app. + deploymentFile: process.env.DEPLOYMENT_FILE, + sentry: { dsn: process.env.SENTRY_DSN }, From 32184330d92c9c92aa199e8b05f16a84cf98d8d1 Mon Sep 17 00:00:00 2001 From: Brian Gough Date: Mon, 22 Mar 2021 11:15:59 +0000 Subject: [PATCH 468/491] delay closing by 1 minute for deployment flip --- services/real-time/app.js | 2 +- services/real-time/app/js/DeploymentManager.js | 6 ++++++ 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/services/real-time/app.js b/services/real-time/app.js index 832bf52a53..bde00b2e17 100644 --- a/services/real-time/app.js +++ b/services/real-time/app.js @@ -85,7 +85,7 @@ io.configure(function () { // a 200 response on '/' is required for load balancer health checks // these operate separately from kubernetes readiness checks app.get('/', function (req, res) { - if (Settings.serviceIsClosed || Settings.shutDownInProgress) { + if (Settings.shutDownInProgress || DeploymentManager.deploymentIsClosed()) { res.sendStatus(503) // Service unavailable } else { res.send('real-time is open') diff --git a/services/real-time/app/js/DeploymentManager.js b/services/real-time/app/js/DeploymentManager.js index a86db02dd1..a3201e6b21 100644 --- a/services/real-time/app/js/DeploymentManager.js +++ b/services/real-time/app/js/DeploymentManager.js @@ -9,10 +9,13 @@ const FILE_CHECK_INTERVAL = 5000 const statusFile = settings.deploymentFile const deploymentColour = settings.deploymentColour +var serviceCloseTime + function updateDeploymentStatus(fileContent) { const closed = fileContent && fileContent.indexOf(deploymentColour) === -1 if (closed && !settings.serviceIsClosed) { settings.serviceIsClosed = true + serviceCloseTime = Date.now() + 60 * 1000 // delay closing by 1 minute logger.warn({ fileContent }, 'closing service') } else if (!closed && settings.serviceIsClosed) { settings.serviceIsClosed = false @@ -49,5 +52,8 @@ module.exports = { checkStatusFileSync() // perform an initial synchronous check at start up setInterval(pollStatusFile, FILE_CHECK_INTERVAL) // continue checking periodically } + }, + deploymentIsClosed() { + return settings.serviceIsClosed && Date.now() > serviceCloseTime } } From 5d858736b1c804e06ea176166b3d17ad0058bb2f Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 25 Mar 2021 11:56:16 +0000 Subject: [PATCH 469/491] Bump ini from 1.3.5 to 1.3.8 Bumps [ini](https://github.com/isaacs/ini) from 1.3.5 to 1.3.8. - [Release notes](https://github.com/isaacs/ini/releases) - [Commits](https://github.com/isaacs/ini/compare/v1.3.5...v1.3.8) Signed-off-by: dependabot[bot] --- services/real-time/package-lock.json | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/services/real-time/package-lock.json b/services/real-time/package-lock.json index 4ba3fdc753..eeff63af28 100644 --- a/services/real-time/package-lock.json +++ b/services/real-time/package-lock.json @@ -3109,9 +3109,9 @@ "integrity": "sha1-Yzwsg+PaQqUC9SRmAiSA9CCCYd4=" }, "ini": { - "version": "1.3.5", - "resolved": "https://registry.npmjs.org/ini/-/ini-1.3.5.tgz", - "integrity": "sha512-RZY5huIKCMRWDUqZlEi72f/lmXKMvuszcMBduliQ3nnWbx9X/ZBQO7DijMEYS9EhHBb2qacRUMtC7svLwe0lcw==" + "version": "1.3.8", + "resolved": "https://registry.npmjs.org/ini/-/ini-1.3.8.tgz", + "integrity": "sha512-JV/yugV2uzW5iMRSiZAyDtQd+nxtUnjeLt0acNdw98kKLrvuRVyB80tsREOE7yvGVgalhZ6RNXCmEHkUKBKxew==" }, "inquirer": { "version": "7.2.0", From 61f43bf5d850c40dc49a24ba28d587572d1769f8 Mon Sep 17 00:00:00 2001 From: Eric Mc Sween Date: Thu, 25 Mar 2021 08:10:43 -0400 Subject: [PATCH 470/491] Upgrade metrics module to 3.5.1 --- services/real-time/package-lock.json | 892 +++++++++++++++------------ services/real-time/package.json | 2 +- 2 files changed, 501 insertions(+), 393 deletions(-) diff --git a/services/real-time/package-lock.json b/services/real-time/package-lock.json index 4ba3fdc753..3a55a51434 100644 --- a/services/real-time/package-lock.json +++ b/services/real-time/package-lock.json @@ -30,6 +30,99 @@ "js-tokens": "^4.0.0" } }, + "@google-cloud/common": { + "version": "3.6.0", + "resolved": "https://registry.npmjs.org/@google-cloud/common/-/common-3.6.0.tgz", + "integrity": "sha512-aHIFTqJZmeTNO9md8XxV+ywuvXF3xBm5WNmgWeeCK+XN5X+kGW0WEX94wGwj+/MdOnrVf4dL2RvSIt9J5yJG6Q==", + "requires": { + "@google-cloud/projectify": "^2.0.0", + "@google-cloud/promisify": "^2.0.0", + "arrify": "^2.0.1", + "duplexify": "^4.1.1", + "ent": "^2.2.0", + "extend": "^3.0.2", + "google-auth-library": "^7.0.2", + "retry-request": "^4.1.1", + "teeny-request": "^7.0.0" + }, + "dependencies": { + "duplexify": { + "version": "4.1.1", + "resolved": "https://registry.npmjs.org/duplexify/-/duplexify-4.1.1.tgz", + "integrity": "sha512-DY3xVEmVHTv1wSzKNbwoU6nVjzI369Y6sPoqfYr0/xlx3IdX2n94xIszTcjPO8W8ZIv0Wb0PXNcjuZyT4wiICA==", + "requires": { + "end-of-stream": "^1.4.1", + "inherits": "^2.0.3", + "readable-stream": "^3.1.1", + "stream-shift": "^1.0.0" + } + }, + "readable-stream": { + "version": "3.6.0", + "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-3.6.0.tgz", + "integrity": "sha512-BViHy7LKeTz4oNnkcLJ+lVSL6vpiFeX6/d3oSH8zCW7UxP2onchk+vTGB143xuFjHS3deTgkKoXXymXqymiIdA==", + "requires": { + "inherits": "^2.0.3", + "string_decoder": "^1.1.1", + "util-deprecate": "^1.0.1" + } + } + } + }, + "@google-cloud/debug-agent": { + "version": "5.1.3", + "resolved": "https://registry.npmjs.org/@google-cloud/debug-agent/-/debug-agent-5.1.3.tgz", + "integrity": "sha512-WbzeEz4MvPlM7DX2QBsPcWgF62u7LSQv/oMYPl0L+TddTebqjDKiVXwxpzWk61NIfcKiet3dyCbPIt3N5o8XPQ==", + "requires": { + "@google-cloud/common": "^3.0.0", + "acorn": "^8.0.0", + "coffeescript": "^2.0.0", + "console-log-level": "^1.4.0", + "extend": "^3.0.2", + "findit2": "^2.2.3", + "gcp-metadata": "^4.0.0", + "p-limit": "^3.0.1", + "semver": "^7.0.0", + "source-map": "^0.6.1", + "split": "^1.0.0" + }, + "dependencies": { + "acorn": { + "version": "8.1.0", + "resolved": "https://registry.npmjs.org/acorn/-/acorn-8.1.0.tgz", + "integrity": "sha512-LWCF/Wn0nfHOmJ9rzQApGnxnvgfROzGilS8936rqN/lfcYkY9MYZzdMqN+2NJ4SlTc+m5HiSa+kNfDtI64dwUA==" + }, + "lru-cache": { + "version": "6.0.0", + "resolved": "https://registry.npmjs.org/lru-cache/-/lru-cache-6.0.0.tgz", + "integrity": "sha512-Jo6dJ04CmSjuznwJSS3pUeWmd/H0ffTlkXXgwZi+eq1UCmqQwCh+eLsYOYCwY991i2Fah4h1BEMCx4qThGbsiA==", + "requires": { + "yallist": "^4.0.0" + } + }, + "p-limit": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/p-limit/-/p-limit-3.1.0.tgz", + "integrity": "sha512-TYOanM3wGwNGsZN2cVTYPArw454xnXj5qmWF1bEoAc4+cU/ol7GVh7odevjp1FNHduHc3KZMcFduxU5Xc6uJRQ==", + "requires": { + "yocto-queue": "^0.1.0" + } + }, + "semver": { + "version": "7.3.5", + "resolved": "https://registry.npmjs.org/semver/-/semver-7.3.5.tgz", + "integrity": "sha512-PoeGJYh8HK4BTO/a9Tf6ZG3veo/A7ZVsYrSA6J8ny9nb3B1VrpkuN+z9OE5wfE5p6H4LchYZsegiQgbJD94ZFQ==", + "requires": { + "lru-cache": "^6.0.0" + } + }, + "yallist": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/yallist/-/yallist-4.0.0.tgz", + "integrity": "sha512-3wdGidZyq5PB084XLES5TpOSRA3wjXAlIWMhum2kRcv/41Sn2emQ0dycQW4uZXLejwKvg6EsvbdlVL+FYEct7A==" + } + } + }, "@google-cloud/logging": { "version": "7.3.0", "resolved": "https://registry.npmjs.org/@google-cloud/logging/-/logging-7.3.0.tgz", @@ -377,6 +470,152 @@ "extend": "^3.0.2" } }, + "@google-cloud/profiler": { + "version": "4.1.1", + "resolved": "https://registry.npmjs.org/@google-cloud/profiler/-/profiler-4.1.1.tgz", + "integrity": "sha512-qk08aDxTaLnu+NoNEh5Jh+Fs5iR8lRLMr5Mb3YJDoZw72jHJI4f5N5F2JWt1xRc9D6da4gA6stBUJrbfbubvGQ==", + "requires": { + "@google-cloud/common": "^3.0.0", + "@types/console-log-level": "^1.4.0", + "@types/semver": "^7.0.0", + "console-log-level": "^1.4.0", + "delay": "^5.0.0", + "extend": "^3.0.2", + "gcp-metadata": "^4.0.0", + "parse-duration": "^1.0.0", + "pprof": "3.0.0", + "pretty-ms": "^7.0.0", + "protobufjs": "~6.10.0", + "semver": "^7.0.0", + "teeny-request": "^7.0.0" + }, + "dependencies": { + "lru-cache": { + "version": "6.0.0", + "resolved": "https://registry.npmjs.org/lru-cache/-/lru-cache-6.0.0.tgz", + "integrity": "sha512-Jo6dJ04CmSjuznwJSS3pUeWmd/H0ffTlkXXgwZi+eq1UCmqQwCh+eLsYOYCwY991i2Fah4h1BEMCx4qThGbsiA==", + "requires": { + "yallist": "^4.0.0" + } + }, + "protobufjs": { + "version": "6.10.2", + "resolved": "https://registry.npmjs.org/protobufjs/-/protobufjs-6.10.2.tgz", + "integrity": "sha512-27yj+04uF6ya9l+qfpH187aqEzfCF4+Uit0I9ZBQVqK09hk/SQzKa2MUqUpXaVa7LOFRg1TSSr3lVxGOk6c0SQ==", + "requires": { + "@protobufjs/aspromise": "^1.1.2", + "@protobufjs/base64": "^1.1.2", + "@protobufjs/codegen": "^2.0.4", + "@protobufjs/eventemitter": "^1.1.0", + "@protobufjs/fetch": "^1.1.0", + "@protobufjs/float": "^1.0.2", + "@protobufjs/inquire": "^1.1.0", + "@protobufjs/path": "^1.1.2", + "@protobufjs/pool": "^1.1.0", + "@protobufjs/utf8": "^1.1.0", + "@types/long": "^4.0.1", + "@types/node": "^13.7.0", + "long": "^4.0.0" + } + }, + "semver": { + "version": "7.3.5", + "resolved": "https://registry.npmjs.org/semver/-/semver-7.3.5.tgz", + "integrity": "sha512-PoeGJYh8HK4BTO/a9Tf6ZG3veo/A7ZVsYrSA6J8ny9nb3B1VrpkuN+z9OE5wfE5p6H4LchYZsegiQgbJD94ZFQ==", + "requires": { + "lru-cache": "^6.0.0" + } + }, + "yallist": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/yallist/-/yallist-4.0.0.tgz", + "integrity": "sha512-3wdGidZyq5PB084XLES5TpOSRA3wjXAlIWMhum2kRcv/41Sn2emQ0dycQW4uZXLejwKvg6EsvbdlVL+FYEct7A==" + } + } + }, + "@google-cloud/projectify": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/@google-cloud/projectify/-/projectify-2.0.1.tgz", + "integrity": "sha512-ZDG38U/Yy6Zr21LaR3BTiiLtpJl6RkPS/JwoRT453G+6Q1DhlV0waNf8Lfu+YVYGIIxgKnLayJRfYlFJfiI8iQ==" + }, + "@google-cloud/promisify": { + "version": "2.0.3", + "resolved": "https://registry.npmjs.org/@google-cloud/promisify/-/promisify-2.0.3.tgz", + "integrity": "sha512-d4VSA86eL/AFTe5xtyZX+ePUjE8dIFu2T8zmdeNBSa5/kNgXPCx/o/wbFNHAGLJdGnk1vddRuMESD9HbOC8irw==" + }, + "@google-cloud/trace-agent": { + "version": "5.1.3", + "resolved": "https://registry.npmjs.org/@google-cloud/trace-agent/-/trace-agent-5.1.3.tgz", + "integrity": "sha512-f+5DX7n6QpDlHA+4kr81z69SLAdrlvd9T8skqCMgnYvtXx14AwzXZyzEDf3jppOYzYoqPPJv8XYiyYHHmYD0BA==", + "requires": { + "@google-cloud/common": "^3.0.0", + "@opencensus/propagation-stackdriver": "0.0.22", + "builtin-modules": "^3.0.0", + "console-log-level": "^1.4.0", + "continuation-local-storage": "^3.2.1", + "extend": "^3.0.2", + "gcp-metadata": "^4.0.0", + "google-auth-library": "^7.0.0", + "hex2dec": "^1.0.1", + "is": "^3.2.0", + "methods": "^1.1.1", + "require-in-the-middle": "^5.0.0", + "semver": "^7.0.0", + "shimmer": "^1.2.0", + "source-map-support": "^0.5.16", + "uuid": "^8.0.0" + }, + "dependencies": { + "@opencensus/core": { + "version": "0.0.22", + "resolved": "https://registry.npmjs.org/@opencensus/core/-/core-0.0.22.tgz", + "integrity": "sha512-ErazJtivjceNoOZI1bG9giQ6cWS45J4i6iPUtlp7dLNu58OLs/v+CD0FsaPCh47XgPxAI12vbBE8Ec09ViwHNA==", + "requires": { + "continuation-local-storage": "^3.2.1", + "log-driver": "^1.2.7", + "semver": "^7.0.0", + "shimmer": "^1.2.0", + "uuid": "^8.0.0" + } + }, + "@opencensus/propagation-stackdriver": { + "version": "0.0.22", + "resolved": "https://registry.npmjs.org/@opencensus/propagation-stackdriver/-/propagation-stackdriver-0.0.22.tgz", + "integrity": "sha512-eBvf/ihb1mN8Yz/ASkz8nHzuMKqygu77+VNnUeR0yEh3Nj+ykB8VVR6lK+NAFXo1Rd1cOsTmgvuXAZgDAGleQQ==", + "requires": { + "@opencensus/core": "^0.0.22", + "hex2dec": "^1.0.1", + "uuid": "^8.0.0" + } + }, + "lru-cache": { + "version": "6.0.0", + "resolved": "https://registry.npmjs.org/lru-cache/-/lru-cache-6.0.0.tgz", + "integrity": "sha512-Jo6dJ04CmSjuznwJSS3pUeWmd/H0ffTlkXXgwZi+eq1UCmqQwCh+eLsYOYCwY991i2Fah4h1BEMCx4qThGbsiA==", + "requires": { + "yallist": "^4.0.0" + } + }, + "semver": { + "version": "7.3.5", + "resolved": "https://registry.npmjs.org/semver/-/semver-7.3.5.tgz", + "integrity": "sha512-PoeGJYh8HK4BTO/a9Tf6ZG3veo/A7ZVsYrSA6J8ny9nb3B1VrpkuN+z9OE5wfE5p6H4LchYZsegiQgbJD94ZFQ==", + "requires": { + "lru-cache": "^6.0.0" + } + }, + "uuid": { + "version": "8.3.2", + "resolved": "https://registry.npmjs.org/uuid/-/uuid-8.3.2.tgz", + "integrity": "sha512-+NYs2QeMWy+GWFOEm9xnn6HCDp0l7QBD7ml8zLUmJ+93Q5NF0NocErnwkTkXVFNiX3/fpC6afS8Dhb/gz7R7eg==" + }, + "yallist": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/yallist/-/yallist-4.0.0.tgz", + "integrity": "sha512-3wdGidZyq5PB084XLES5TpOSRA3wjXAlIWMhum2kRcv/41Sn2emQ0dycQW4uZXLejwKvg6EsvbdlVL+FYEct7A==" + } + } + }, "@grpc/grpc-js": { "version": "1.0.5", "resolved": "https://registry.npmjs.org/@grpc/grpc-js/-/grpc-js-1.0.5.tgz", @@ -431,9 +670,9 @@ } }, "@overleaf/metrics": { - "version": "3.4.1", - "resolved": "https://registry.npmjs.org/@overleaf/metrics/-/metrics-3.4.1.tgz", - "integrity": "sha512-OgjlzuC+2gPdIEDHhmd9LDMu01tk1ln0cJhw1727BZ+Wgf2Z1hjuHRt4JeCkf+PFTHwJutVYT8v6IGPpNEPtbg==", + "version": "3.5.1", + "resolved": "https://registry.npmjs.org/@overleaf/metrics/-/metrics-3.5.1.tgz", + "integrity": "sha512-RLHxkMF7Y3725L3QwXo9cIn2gGobsMYUGuxKxg7PVMrPTMsomHEMeG7StOxCO7ML1Z/BwB/9nsVYNrsRdAJtKg==", "requires": { "@google-cloud/debug-agent": "^5.1.2", "@google-cloud/profiler": "^4.0.3", @@ -444,369 +683,10 @@ "yn": "^3.1.1" }, "dependencies": { - "@google-cloud/common": { - "version": "3.5.0", - "resolved": "https://registry.npmjs.org/@google-cloud/common/-/common-3.5.0.tgz", - "integrity": "sha512-10d7ZAvKhq47L271AqvHEd8KzJqGU45TY+rwM2Z3JHuB070FeTi7oJJd7elfrnKaEvaktw3hH2wKnRWxk/3oWQ==", - "requires": { - "@google-cloud/projectify": "^2.0.0", - "@google-cloud/promisify": "^2.0.0", - "arrify": "^2.0.1", - "duplexify": "^4.1.1", - "ent": "^2.2.0", - "extend": "^3.0.2", - "google-auth-library": "^6.1.1", - "retry-request": "^4.1.1", - "teeny-request": "^7.0.0" - } - }, - "@google-cloud/debug-agent": { - "version": "5.1.3", - "resolved": "https://registry.npmjs.org/@google-cloud/debug-agent/-/debug-agent-5.1.3.tgz", - "integrity": "sha512-WbzeEz4MvPlM7DX2QBsPcWgF62u7LSQv/oMYPl0L+TddTebqjDKiVXwxpzWk61NIfcKiet3dyCbPIt3N5o8XPQ==", - "requires": { - "@google-cloud/common": "^3.0.0", - "acorn": "^8.0.0", - "coffeescript": "^2.0.0", - "console-log-level": "^1.4.0", - "extend": "^3.0.2", - "findit2": "^2.2.3", - "gcp-metadata": "^4.0.0", - "p-limit": "^3.0.1", - "semver": "^7.0.0", - "source-map": "^0.6.1", - "split": "^1.0.0" - } - }, - "@google-cloud/profiler": { - "version": "4.1.0", - "resolved": "https://registry.npmjs.org/@google-cloud/profiler/-/profiler-4.1.0.tgz", - "integrity": "sha512-9e1zXRctLSUHAoAsFGwE4rS28fr0siiG+jXl5OpwTK8ZAUlxb70aosHaZGdsv8YXrYKjuiufjRZ/OXCs0XLI9g==", - "requires": { - "@google-cloud/common": "^3.0.0", - "@types/console-log-level": "^1.4.0", - "@types/semver": "^7.0.0", - "console-log-level": "^1.4.0", - "delay": "^4.0.1", - "extend": "^3.0.2", - "gcp-metadata": "^4.0.0", - "parse-duration": "^0.4.4", - "pprof": "3.0.0", - "pretty-ms": "^7.0.0", - "protobufjs": "~6.10.0", - "semver": "^7.0.0", - "teeny-request": "^7.0.0" - } - }, - "@google-cloud/projectify": { - "version": "2.0.1", - "resolved": "https://registry.npmjs.org/@google-cloud/projectify/-/projectify-2.0.1.tgz", - "integrity": "sha512-ZDG38U/Yy6Zr21LaR3BTiiLtpJl6RkPS/JwoRT453G+6Q1DhlV0waNf8Lfu+YVYGIIxgKnLayJRfYlFJfiI8iQ==" - }, - "@google-cloud/promisify": { - "version": "2.0.3", - "resolved": "https://registry.npmjs.org/@google-cloud/promisify/-/promisify-2.0.3.tgz", - "integrity": "sha512-d4VSA86eL/AFTe5xtyZX+ePUjE8dIFu2T8zmdeNBSa5/kNgXPCx/o/wbFNHAGLJdGnk1vddRuMESD9HbOC8irw==" - }, - "@google-cloud/trace-agent": { - "version": "5.1.1", - "resolved": "https://registry.npmjs.org/@google-cloud/trace-agent/-/trace-agent-5.1.1.tgz", - "integrity": "sha512-YTcK0RLN90pLCprg0XC8uV4oAVd79vsXhkcxmEVwiOOYjUDvSrAhb7y/0SY606zgfhJHmUTNb/fZSWEtZP/slQ==", - "requires": { - "@google-cloud/common": "^3.0.0", - "@opencensus/propagation-stackdriver": "0.0.22", - "builtin-modules": "^3.0.0", - "console-log-level": "^1.4.0", - "continuation-local-storage": "^3.2.1", - "extend": "^3.0.2", - "gcp-metadata": "^4.0.0", - "google-auth-library": "^6.0.0", - "hex2dec": "^1.0.1", - "is": "^3.2.0", - "methods": "^1.1.1", - "require-in-the-middle": "^5.0.0", - "semver": "^7.0.0", - "shimmer": "^1.2.0", - "source-map-support": "^0.5.16", - "uuid": "^8.0.0" - } - }, - "@opencensus/core": { - "version": "0.0.22", - "resolved": "https://registry.npmjs.org/@opencensus/core/-/core-0.0.22.tgz", - "integrity": "sha512-ErazJtivjceNoOZI1bG9giQ6cWS45J4i6iPUtlp7dLNu58OLs/v+CD0FsaPCh47XgPxAI12vbBE8Ec09ViwHNA==", - "requires": { - "continuation-local-storage": "^3.2.1", - "log-driver": "^1.2.7", - "semver": "^7.0.0", - "shimmer": "^1.2.0", - "uuid": "^8.0.0" - } - }, - "@opencensus/propagation-stackdriver": { - "version": "0.0.22", - "resolved": "https://registry.npmjs.org/@opencensus/propagation-stackdriver/-/propagation-stackdriver-0.0.22.tgz", - "integrity": "sha512-eBvf/ihb1mN8Yz/ASkz8nHzuMKqygu77+VNnUeR0yEh3Nj+ykB8VVR6lK+NAFXo1Rd1cOsTmgvuXAZgDAGleQQ==", - "requires": { - "@opencensus/core": "^0.0.22", - "hex2dec": "^1.0.1", - "uuid": "^8.0.0" - } - }, - "@types/semver": { - "version": "7.3.4", - "resolved": "https://registry.npmjs.org/@types/semver/-/semver-7.3.4.tgz", - "integrity": "sha512-+nVsLKlcUCeMzD2ufHEYuJ9a2ovstb6Dp52A5VsoKxDXgvE051XgHI/33I1EymwkRGQkwnA0LkhnUzituGs4EQ==" - }, - "acorn": { - "version": "8.0.4", - "resolved": "https://registry.npmjs.org/acorn/-/acorn-8.0.4.tgz", - "integrity": "sha512-XNP0PqF1XD19ZlLKvB7cMmnZswW4C/03pRHgirB30uSJTaS3A3V1/P4sS3HPvFmjoriPCJQs+JDSbm4bL1TxGQ==" - }, - "agent-base": { - "version": "6.0.2", - "resolved": "https://registry.npmjs.org/agent-base/-/agent-base-6.0.2.tgz", - "integrity": "sha512-RZNwNclF7+MS/8bDg70amg32dyeZGZxiDuQmZxKLAlQjr3jGyLx+4Kkk58UO7D2QdgFIQCovuSuZESne6RG6XQ==", - "requires": { - "debug": "4" - } - }, - "bignumber.js": { - "version": "9.0.1", - "resolved": "https://registry.npmjs.org/bignumber.js/-/bignumber.js-9.0.1.tgz", - "integrity": "sha512-IdZR9mh6ahOBv/hYGiXyVuyCetmGJhtYkqLBpTStdhEGjegpPlUawydyaF3pbIOFynJTpllEs+NP+CS9jKFLjA==" - }, - "coffeescript": { - "version": "2.5.1", - "resolved": "https://registry.npmjs.org/coffeescript/-/coffeescript-2.5.1.tgz", - "integrity": "sha512-J2jRPX0eeFh5VKyVnoLrfVFgLZtnnmp96WQSLAS8OrLm2wtQLcnikYKe1gViJKDH7vucjuhHvBKKBP3rKcD1tQ==" - }, - "debug": { - "version": "4.3.1", - "resolved": "https://registry.npmjs.org/debug/-/debug-4.3.1.tgz", - "integrity": "sha512-doEwdvm4PCeK4K3RQN2ZC2BYUBaxwLARCqZmMjtF8a51J2Rb0xpVloFRnCODwqjpwnAoao4pelN8l3RJdv3gRQ==", - "requires": { - "ms": "2.1.2" - } - }, - "duplexify": { - "version": "4.1.1", - "resolved": "https://registry.npmjs.org/duplexify/-/duplexify-4.1.1.tgz", - "integrity": "sha512-DY3xVEmVHTv1wSzKNbwoU6nVjzI369Y6sPoqfYr0/xlx3IdX2n94xIszTcjPO8W8ZIv0Wb0PXNcjuZyT4wiICA==", - "requires": { - "end-of-stream": "^1.4.1", - "inherits": "^2.0.3", - "readable-stream": "^3.1.1", - "stream-shift": "^1.0.0" - } - }, - "gaxios": { - "version": "4.0.1", - "resolved": "https://registry.npmjs.org/gaxios/-/gaxios-4.0.1.tgz", - "integrity": "sha512-jOin8xRZ/UytQeBpSXFqIzqU7Fi5TqgPNLlUsSB8kjJ76+FiGBfImF8KJu++c6J4jOldfJUtt0YmkRj2ZpSHTQ==", - "requires": { - "abort-controller": "^3.0.0", - "extend": "^3.0.2", - "https-proxy-agent": "^5.0.0", - "is-stream": "^2.0.0", - "node-fetch": "^2.3.0" - } - }, - "gcp-metadata": { - "version": "4.2.1", - "resolved": "https://registry.npmjs.org/gcp-metadata/-/gcp-metadata-4.2.1.tgz", - "integrity": "sha512-tSk+REe5iq/N+K+SK1XjZJUrFPuDqGZVzCy2vocIHIGmPlTGsa8owXMJwGkrXr73NO0AzhPW4MF2DEHz7P2AVw==", - "requires": { - "gaxios": "^4.0.0", - "json-bigint": "^1.0.0" - } - }, - "google-auth-library": { - "version": "6.1.3", - "resolved": "https://registry.npmjs.org/google-auth-library/-/google-auth-library-6.1.3.tgz", - "integrity": "sha512-m9mwvY3GWbr7ZYEbl61isWmk+fvTmOt0YNUfPOUY2VH8K5pZlAIWJjxEi0PqR3OjMretyiQLI6GURMrPSwHQ2g==", - "requires": { - "arrify": "^2.0.0", - "base64-js": "^1.3.0", - "ecdsa-sig-formatter": "^1.0.11", - "fast-text-encoding": "^1.0.0", - "gaxios": "^4.0.0", - "gcp-metadata": "^4.2.0", - "gtoken": "^5.0.4", - "jws": "^4.0.0", - "lru-cache": "^6.0.0" - } - }, - "google-p12-pem": { - "version": "3.0.3", - "resolved": "https://registry.npmjs.org/google-p12-pem/-/google-p12-pem-3.0.3.tgz", - "integrity": "sha512-wS0ek4ZtFx/ACKYF3JhyGe5kzH7pgiQ7J5otlumqR9psmWMYc+U9cErKlCYVYHoUaidXHdZ2xbo34kB+S+24hA==", - "requires": { - "node-forge": "^0.10.0" - } - }, - "gtoken": { - "version": "5.1.0", - "resolved": "https://registry.npmjs.org/gtoken/-/gtoken-5.1.0.tgz", - "integrity": "sha512-4d8N6Lk8TEAHl9vVoRVMh9BNOKWVgl2DdNtr3428O75r3QFrF/a5MMu851VmK0AA8+iSvbwRv69k5XnMLURGhg==", - "requires": { - "gaxios": "^4.0.0", - "google-p12-pem": "^3.0.3", - "jws": "^4.0.0", - "mime": "^2.2.0" - } - }, - "https-proxy-agent": { - "version": "5.0.0", - "resolved": "https://registry.npmjs.org/https-proxy-agent/-/https-proxy-agent-5.0.0.tgz", - "integrity": "sha512-EkYm5BcKUGiduxzSt3Eppko+PiNWNEpa4ySk9vTC6wDsQJW9rHSa+UhGNJoRYp7bz6Ht1eaRIa6QaJqO5rCFbA==", - "requires": { - "agent-base": "6", - "debug": "4" - } - }, - "json-bigint": { - "version": "1.0.0", - "resolved": "https://registry.npmjs.org/json-bigint/-/json-bigint-1.0.0.tgz", - "integrity": "sha512-SiPv/8VpZuWbvLSMtTDU8hEfrZWg/mH/nV/b4o0CYbSxu1UIQPLdwKOCIyLQX+VIPO5vrLX3i8qtqFyhdPSUSQ==", - "requires": { - "bignumber.js": "^9.0.0" - } - }, - "jwa": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/jwa/-/jwa-2.0.0.tgz", - "integrity": "sha512-jrZ2Qx916EA+fq9cEAeCROWPTfCwi1IVHqT2tapuqLEVVDKFDENFw1oL+MwrTvH6msKxsd1YTDVw6uKEcsrLEA==", - "requires": { - "buffer-equal-constant-time": "1.0.1", - "ecdsa-sig-formatter": "1.0.11", - "safe-buffer": "^5.0.1" - } - }, - "jws": { - "version": "4.0.0", - "resolved": "https://registry.npmjs.org/jws/-/jws-4.0.0.tgz", - "integrity": "sha512-KDncfTmOZoOMTFG4mBlG0qUIOlc03fmzH+ru6RgYVZhPkyiy/92Owlt/8UEN+a4TXR1FQetfIpJE8ApdvdVxTg==", - "requires": { - "jwa": "^2.0.0", - "safe-buffer": "^5.0.1" - } - }, - "lru-cache": { - "version": "6.0.0", - "resolved": "https://registry.npmjs.org/lru-cache/-/lru-cache-6.0.0.tgz", - "integrity": "sha512-Jo6dJ04CmSjuznwJSS3pUeWmd/H0ffTlkXXgwZi+eq1UCmqQwCh+eLsYOYCwY991i2Fah4h1BEMCx4qThGbsiA==", - "requires": { - "yallist": "^4.0.0" - } - }, - "mime": { - "version": "2.4.6", - "resolved": "https://registry.npmjs.org/mime/-/mime-2.4.6.tgz", - "integrity": "sha512-RZKhC3EmpBchfTGBVb8fb+RL2cWyw/32lshnsETttkBAyAUXSGHxbEJWWRXc751DrIxG1q04b8QwMbAwkRPpUA==" - }, - "ms": { - "version": "2.1.2", - "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.2.tgz", - "integrity": "sha512-sGkPx+VjMtmA6MX27oA4FBFELFCZZ4S4XqeGOXCv68tT+jb3vk/RyaKWP0PTKyWtmLSM0b+adUTEvbs1PEaH2w==" - }, - "node-forge": { - "version": "0.10.0", - "resolved": "https://registry.npmjs.org/node-forge/-/node-forge-0.10.0.tgz", - "integrity": "sha512-PPmu8eEeG9saEUvI97fm4OYxXVB6bFvyNTyiUOBichBpFG8A1Ljw3bY62+5oOjDEMHRnd0Y7HQ+x7uzxOzC6JA==" - }, - "p-limit": { - "version": "3.1.0", - "resolved": "https://registry.npmjs.org/p-limit/-/p-limit-3.1.0.tgz", - "integrity": "sha512-TYOanM3wGwNGsZN2cVTYPArw454xnXj5qmWF1bEoAc4+cU/ol7GVh7odevjp1FNHduHc3KZMcFduxU5Xc6uJRQ==", - "requires": { - "yocto-queue": "^0.1.0" - } - }, - "parse-duration": { - "version": "0.4.4", - "resolved": "https://registry.npmjs.org/parse-duration/-/parse-duration-0.4.4.tgz", - "integrity": "sha512-KbAJuYGUhZkB9gotDiKLnZ7Z3VTacK3fgwmDdB6ZVDtJbMBT6MfLga0WJaYpPDu0mzqT0NgHtHDt5PY4l0nidg==" - }, - "pretty-ms": { - "version": "7.0.1", - "resolved": "https://registry.npmjs.org/pretty-ms/-/pretty-ms-7.0.1.tgz", - "integrity": "sha512-973driJZvxiGOQ5ONsFhOF/DtzPMOMtgC11kCpUrPGMTgqp2q/1gwzCquocrN33is0VZ5GFHXZYMM9l6h67v2Q==", - "requires": { - "parse-ms": "^2.1.0" - } - }, - "protobufjs": { - "version": "6.10.2", - "resolved": "https://registry.npmjs.org/protobufjs/-/protobufjs-6.10.2.tgz", - "integrity": "sha512-27yj+04uF6ya9l+qfpH187aqEzfCF4+Uit0I9ZBQVqK09hk/SQzKa2MUqUpXaVa7LOFRg1TSSr3lVxGOk6c0SQ==", - "requires": { - "@protobufjs/aspromise": "^1.1.2", - "@protobufjs/base64": "^1.1.2", - "@protobufjs/codegen": "^2.0.4", - "@protobufjs/eventemitter": "^1.1.0", - "@protobufjs/fetch": "^1.1.0", - "@protobufjs/float": "^1.0.2", - "@protobufjs/inquire": "^1.1.0", - "@protobufjs/path": "^1.1.2", - "@protobufjs/pool": "^1.1.0", - "@protobufjs/utf8": "^1.1.0", - "@types/long": "^4.0.1", - "@types/node": "^13.7.0", - "long": "^4.0.0" - } - }, - "readable-stream": { - "version": "3.6.0", - "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-3.6.0.tgz", - "integrity": "sha512-BViHy7LKeTz4oNnkcLJ+lVSL6vpiFeX6/d3oSH8zCW7UxP2onchk+vTGB143xuFjHS3deTgkKoXXymXqymiIdA==", - "requires": { - "inherits": "^2.0.3", - "string_decoder": "^1.1.1", - "util-deprecate": "^1.0.1" - } - }, - "require-in-the-middle": { - "version": "5.0.3", - "resolved": "https://registry.npmjs.org/require-in-the-middle/-/require-in-the-middle-5.0.3.tgz", - "integrity": "sha512-p/ICV8uMlqC4tjOYabLMxAWCIKa0YUQgZZ6KDM0xgXJNgdGQ1WmL2A07TwmrZw+wi6ITUFKzH5v3n+ENEyXVkA==", - "requires": { - "debug": "^4.1.1", - "module-details-from-path": "^1.0.3", - "resolve": "^1.12.0" - } - }, - "semver": { - "version": "7.3.2", - "resolved": "https://registry.npmjs.org/semver/-/semver-7.3.2.tgz", - "integrity": "sha512-OrOb32TeeambH6UrhtShmF7CRDqhL6/5XpPNp2DuRH6+9QLw/orhp72j87v8Qa1ScDkvrrBNpZcDejAirJmfXQ==" - }, - "teeny-request": { - "version": "7.0.1", - "resolved": "https://registry.npmjs.org/teeny-request/-/teeny-request-7.0.1.tgz", - "integrity": "sha512-sasJmQ37klOlplL4Ia/786M5YlOcoLGQyq2TE4WHSRupbAuDaQW0PfVxV4MtdBtRJ4ngzS+1qim8zP6Zp35qCw==", - "requires": { - "http-proxy-agent": "^4.0.0", - "https-proxy-agent": "^5.0.0", - "node-fetch": "^2.6.1", - "stream-events": "^1.0.5", - "uuid": "^8.0.0" - } - }, "underscore": { "version": "1.6.0", "resolved": "https://registry.npmjs.org/underscore/-/underscore-1.6.0.tgz", "integrity": "sha1-izixDKze9jM3uLJOT/htRa6lKag=" - }, - "uuid": { - "version": "8.3.1", - "resolved": "https://registry.npmjs.org/uuid/-/uuid-8.3.1.tgz", - "integrity": "sha512-FOmRr+FmWEIG8uhZv6C2bTgEVXsHk08kE7mPlrBbEe+c3r9pjceVPgupIfNIhc4yx55H69OXANrUaSuu9eInKg==" - }, - "yallist": { - "version": "4.0.0", - "resolved": "https://registry.npmjs.org/yallist/-/yallist-4.0.0.tgz", - "integrity": "sha512-3wdGidZyq5PB084XLES5TpOSRA3wjXAlIWMhum2kRcv/41Sn2emQ0dycQW4uZXLejwKvg6EsvbdlVL+FYEct7A==" } } }, @@ -980,6 +860,11 @@ "resolved": "https://registry.npmjs.org/@types/node/-/node-13.9.5.tgz", "integrity": "sha512-hkzMMD3xu6BrJpGVLeQ3htQQNAcOrJjX7WFmtK8zWQpz2UJf13LCFF2ALA7c9OVdvc2vQJeDdjfR35M0sBCxvw==" }, + "@types/semver": { + "version": "7.3.4", + "resolved": "https://registry.npmjs.org/@types/semver/-/semver-7.3.4.tgz", + "integrity": "sha512-+nVsLKlcUCeMzD2ufHEYuJ9a2ovstb6Dp52A5VsoKxDXgvE051XgHI/33I1EymwkRGQkwnA0LkhnUzituGs4EQ==" + }, "@typescript-eslint/experimental-utils": { "version": "1.13.0", "resolved": "https://registry.npmjs.org/@typescript-eslint/experimental-utils/-/experimental-utils-1.13.0.tgz", @@ -1096,6 +981,29 @@ "zeparser": "0.0.5" } }, + "agent-base": { + "version": "6.0.2", + "resolved": "https://registry.npmjs.org/agent-base/-/agent-base-6.0.2.tgz", + "integrity": "sha512-RZNwNclF7+MS/8bDg70amg32dyeZGZxiDuQmZxKLAlQjr3jGyLx+4Kkk58UO7D2QdgFIQCovuSuZESne6RG6XQ==", + "requires": { + "debug": "4" + }, + "dependencies": { + "debug": { + "version": "4.3.1", + "resolved": "https://registry.npmjs.org/debug/-/debug-4.3.1.tgz", + "integrity": "sha512-doEwdvm4PCeK4K3RQN2ZC2BYUBaxwLARCqZmMjtF8a51J2Rb0xpVloFRnCODwqjpwnAoao4pelN8l3RJdv3gRQ==", + "requires": { + "ms": "2.1.2" + } + }, + "ms": { + "version": "2.1.2", + "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.2.tgz", + "integrity": "sha512-sGkPx+VjMtmA6MX27oA4FBFELFCZZ4S4XqeGOXCv68tT+jb3vk/RyaKWP0PTKyWtmLSM0b+adUTEvbs1PEaH2w==" + } + } + }, "ajv": { "version": "6.12.2", "resolved": "https://registry.npmjs.org/ajv/-/ajv-6.12.2.tgz", @@ -1312,7 +1220,7 @@ "bindings": { "version": "1.5.0", "resolved": "https://registry.npmjs.org/bindings/-/bindings-1.5.0.tgz", - "integrity": "sha1-EDU8npRTNLwFEabZCzj7x8nFBN8=", + "integrity": "sha512-p2q/t/mhvuOj/UeLlV6566GD/guowlr0hHxClI0W9m7MWYkL1F0hLo+0Aexs9HSPCtR1SXQ0TD3MMKrXZajbiQ==", "requires": { "file-uri-to-path": "1.0.0" } @@ -1380,9 +1288,9 @@ "integrity": "sha512-MQcXEUbCKtEo7bhqEs6560Hyd4XaovZlO/k9V3hjVUF/zwW7KBVdSK4gIt/bzwS9MbR5qob+F5jusZsb0YQK2A==" }, "builtin-modules": { - "version": "3.1.0", - "resolved": "https://registry.npmjs.org/builtin-modules/-/builtin-modules-3.1.0.tgz", - "integrity": "sha1-qtl8FRMet2tltQ7yCOdYTNdqdIQ=" + "version": "3.2.0", + "resolved": "https://registry.npmjs.org/builtin-modules/-/builtin-modules-3.2.0.tgz", + "integrity": "sha512-lGzLKcioL90C7wMczpkY0n/oART3MbBa8R9OFGE1rJxoVI86u4WAGfEk8Wjv10eKSyTHVGkSo3bvBylCEtk7LA==" }, "bunyan": { "version": "0.22.3", @@ -1541,6 +1449,11 @@ "resolved": "https://registry.npmjs.org/code-point-at/-/code-point-at-1.1.0.tgz", "integrity": "sha1-DQcLTQQ6W+ozovGkDi7bPZpMz3c=" }, + "coffeescript": { + "version": "2.5.1", + "resolved": "https://registry.npmjs.org/coffeescript/-/coffeescript-2.5.1.tgz", + "integrity": "sha512-J2jRPX0eeFh5VKyVnoLrfVFgLZtnnmp96WQSLAS8OrLm2wtQLcnikYKe1gViJKDH7vucjuhHvBKKBP3rKcD1tQ==" + }, "color-convert": { "version": "1.9.3", "resolved": "https://registry.npmjs.org/color-convert/-/color-convert-1.9.3.tgz", @@ -1584,9 +1497,9 @@ }, "dependencies": { "mime-db": { - "version": "1.45.0", - "resolved": "https://registry.npmjs.org/mime-db/-/mime-db-1.45.0.tgz", - "integrity": "sha512-CkqLUxUk15hofLoLyljJSrukZi8mAtgd+yE5uO4tqRZsdsAJKv0O+rFMhVDRJgozy+yG6md5KwuXhD4ocIoP+w==" + "version": "1.46.0", + "resolved": "https://registry.npmjs.org/mime-db/-/mime-db-1.46.0.tgz", + "integrity": "sha512-svXaP8UQRZ5K7or+ZmfNhg2xX3yKDMUzqadsSqi4NCH/KomcH75MAMYAGVlvXn4+b/xOPhS3I2uHKRUzvjY7BQ==" } } }, @@ -1648,7 +1561,7 @@ "console-log-level": { "version": "1.4.1", "resolved": "https://registry.npmjs.org/console-log-level/-/console-log-level-1.4.1.tgz", - "integrity": "sha1-nFprue8e9lsFq6gwKLD/iUzfYwo=" + "integrity": "sha512-VZzbIORbP+PPcN/gg3DXClTLPLg5Slwd5fL2MIc+o1qZ4BXBvWyc6QxPk6T/Mkr6IVjRpoAGf32XxP3ZWMVRcQ==" }, "contains-path": { "version": "0.1.0", @@ -1811,9 +1724,9 @@ } }, "delay": { - "version": "4.3.0", - "resolved": "https://registry.npmjs.org/delay/-/delay-4.3.0.tgz", - "integrity": "sha1-7+6/uPVFV5yzlrOnIkQ+yW0UxQ4=" + "version": "5.0.0", + "resolved": "https://registry.npmjs.org/delay/-/delay-5.0.0.tgz", + "integrity": "sha512-ReEBKkIfe4ya47wlPYf/gu5ib6yUG0/Aez0JQZQz94kiWtRQvZIQbTiehsnwHvLSWJnQdhVeqYue7Id1dKr0qw==" }, "delayed-stream": { "version": "1.0.0", @@ -2719,6 +2632,42 @@ } } }, + "gaxios": { + "version": "4.2.0", + "resolved": "https://registry.npmjs.org/gaxios/-/gaxios-4.2.0.tgz", + "integrity": "sha512-Ms7fNifGv0XVU+6eIyL9LB7RVESeML9+cMvkwGS70xyD6w2Z80wl6RiqiJ9k1KFlJCUTQqFFc8tXmPQfSKUe8g==", + "requires": { + "abort-controller": "^3.0.0", + "extend": "^3.0.2", + "https-proxy-agent": "^5.0.0", + "is-stream": "^2.0.0", + "node-fetch": "^2.3.0" + } + }, + "gcp-metadata": { + "version": "4.2.1", + "resolved": "https://registry.npmjs.org/gcp-metadata/-/gcp-metadata-4.2.1.tgz", + "integrity": "sha512-tSk+REe5iq/N+K+SK1XjZJUrFPuDqGZVzCy2vocIHIGmPlTGsa8owXMJwGkrXr73NO0AzhPW4MF2DEHz7P2AVw==", + "requires": { + "gaxios": "^4.0.0", + "json-bigint": "^1.0.0" + }, + "dependencies": { + "bignumber.js": { + "version": "9.0.1", + "resolved": "https://registry.npmjs.org/bignumber.js/-/bignumber.js-9.0.1.tgz", + "integrity": "sha512-IdZR9mh6ahOBv/hYGiXyVuyCetmGJhtYkqLBpTStdhEGjegpPlUawydyaF3pbIOFynJTpllEs+NP+CS9jKFLjA==" + }, + "json-bigint": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/json-bigint/-/json-bigint-1.0.0.tgz", + "integrity": "sha512-SiPv/8VpZuWbvLSMtTDU8hEfrZWg/mH/nV/b4o0CYbSxu1UIQPLdwKOCIyLQX+VIPO5vrLX3i8qtqFyhdPSUSQ==", + "requires": { + "bignumber.js": "^9.0.0" + } + } + } + }, "get-caller-file": { "version": "2.0.5", "resolved": "https://registry.npmjs.org/get-caller-file/-/get-caller-file-2.0.5.tgz", @@ -2770,6 +2719,37 @@ "type-fest": "^0.8.1" } }, + "google-auth-library": { + "version": "7.0.3", + "resolved": "https://registry.npmjs.org/google-auth-library/-/google-auth-library-7.0.3.tgz", + "integrity": "sha512-6wJNYqY1QUr5I2lWaUkkzOT2b9OCNhNQrdFOt/bsBbGb7T7NCdEvrBsXraUm+KTUGk2xGlQ7m9RgUd4Llcw8NQ==", + "requires": { + "arrify": "^2.0.0", + "base64-js": "^1.3.0", + "ecdsa-sig-formatter": "^1.0.11", + "fast-text-encoding": "^1.0.0", + "gaxios": "^4.0.0", + "gcp-metadata": "^4.2.0", + "gtoken": "^5.0.4", + "jws": "^4.0.0", + "lru-cache": "^6.0.0" + }, + "dependencies": { + "lru-cache": { + "version": "6.0.0", + "resolved": "https://registry.npmjs.org/lru-cache/-/lru-cache-6.0.0.tgz", + "integrity": "sha512-Jo6dJ04CmSjuznwJSS3pUeWmd/H0ffTlkXXgwZi+eq1UCmqQwCh+eLsYOYCwY991i2Fah4h1BEMCx4qThGbsiA==", + "requires": { + "yallist": "^4.0.0" + } + }, + "yallist": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/yallist/-/yallist-4.0.0.tgz", + "integrity": "sha512-3wdGidZyq5PB084XLES5TpOSRA3wjXAlIWMhum2kRcv/41Sn2emQ0dycQW4uZXLejwKvg6EsvbdlVL+FYEct7A==" + } + } + }, "google-gax": { "version": "1.15.3", "resolved": "https://registry.npmjs.org/google-gax/-/google-gax-1.15.3.tgz", @@ -2909,6 +2889,14 @@ } } }, + "google-p12-pem": { + "version": "3.0.3", + "resolved": "https://registry.npmjs.org/google-p12-pem/-/google-p12-pem-3.0.3.tgz", + "integrity": "sha512-wS0ek4ZtFx/ACKYF3JhyGe5kzH7pgiQ7J5otlumqR9psmWMYc+U9cErKlCYVYHoUaidXHdZ2xbo34kB+S+24hA==", + "requires": { + "node-forge": "^0.10.0" + } + }, "graceful-fs": { "version": "4.2.4", "resolved": "https://registry.npmjs.org/graceful-fs/-/graceful-fs-4.2.4.tgz", @@ -2921,6 +2909,16 @@ "integrity": "sha512-qBr4OuELkhPenW6goKVXiv47US3clb3/IbuWF9KNKEijAy9oeHxU9IgzjvJhHkUzhaj7rOUD7+YGWqUjLp5oSA==", "dev": true }, + "gtoken": { + "version": "5.2.1", + "resolved": "https://registry.npmjs.org/gtoken/-/gtoken-5.2.1.tgz", + "integrity": "sha512-OY0BfPKe3QnMsY9MzTHTSKn+Vl2l1CcLe6BwDEQj00mbbkl5nyQ/7EUREstg4fQNZ8iYE7br4JJ7TdKeDOPWmw==", + "requires": { + "gaxios": "^4.0.0", + "google-p12-pem": "^3.0.3", + "jws": "^4.0.0" + } + }, "har-schema": { "version": "2.0.0", "resolved": "https://registry.npmjs.org/har-schema/-/har-schema-2.0.0.tgz", @@ -3050,6 +3048,30 @@ "sshpk": "^1.7.0" } }, + "https-proxy-agent": { + "version": "5.0.0", + "resolved": "https://registry.npmjs.org/https-proxy-agent/-/https-proxy-agent-5.0.0.tgz", + "integrity": "sha512-EkYm5BcKUGiduxzSt3Eppko+PiNWNEpa4ySk9vTC6wDsQJW9rHSa+UhGNJoRYp7bz6Ht1eaRIa6QaJqO5rCFbA==", + "requires": { + "agent-base": "6", + "debug": "4" + }, + "dependencies": { + "debug": { + "version": "4.3.1", + "resolved": "https://registry.npmjs.org/debug/-/debug-4.3.1.tgz", + "integrity": "sha512-doEwdvm4PCeK4K3RQN2ZC2BYUBaxwLARCqZmMjtF8a51J2Rb0xpVloFRnCODwqjpwnAoao4pelN8l3RJdv3gRQ==", + "requires": { + "ms": "2.1.2" + } + }, + "ms": { + "version": "2.1.2", + "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.2.tgz", + "integrity": "sha512-sGkPx+VjMtmA6MX27oA4FBFELFCZZ4S4XqeGOXCv68tT+jb3vk/RyaKWP0PTKyWtmLSM0b+adUTEvbs1PEaH2w==" + } + } + }, "iconv-lite": { "version": "0.4.24", "resolved": "https://registry.npmjs.org/iconv-lite/-/iconv-lite-0.4.24.tgz", @@ -3109,9 +3131,9 @@ "integrity": "sha1-Yzwsg+PaQqUC9SRmAiSA9CCCYd4=" }, "ini": { - "version": "1.3.5", - "resolved": "https://registry.npmjs.org/ini/-/ini-1.3.5.tgz", - "integrity": "sha512-RZY5huIKCMRWDUqZlEi72f/lmXKMvuszcMBduliQ3nnWbx9X/ZBQO7DijMEYS9EhHBb2qacRUMtC7svLwe0lcw==" + "version": "1.3.8", + "resolved": "https://registry.npmjs.org/ini/-/ini-1.3.8.tgz", + "integrity": "sha512-JV/yugV2uzW5iMRSiZAyDtQd+nxtUnjeLt0acNdw98kKLrvuRVyB80tsREOE7yvGVgalhZ6RNXCmEHkUKBKxew==" }, "inquirer": { "version": "7.2.0", @@ -3440,6 +3462,25 @@ "integrity": "sha512-aWgeGFW67BP3e5181Ep1Fv2v8z//iBJfrvyTnq8wG86vEESwmonn1zPBJ0VfmT9CJq2FIT0VsETtrNFm2a+SHA==", "dev": true }, + "jwa": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/jwa/-/jwa-2.0.0.tgz", + "integrity": "sha512-jrZ2Qx916EA+fq9cEAeCROWPTfCwi1IVHqT2tapuqLEVVDKFDENFw1oL+MwrTvH6msKxsd1YTDVw6uKEcsrLEA==", + "requires": { + "buffer-equal-constant-time": "1.0.1", + "ecdsa-sig-formatter": "1.0.11", + "safe-buffer": "^5.0.1" + } + }, + "jws": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/jws/-/jws-4.0.0.tgz", + "integrity": "sha512-KDncfTmOZoOMTFG4mBlG0qUIOlc03fmzH+ru6RgYVZhPkyiy/92Owlt/8UEN+a4TXR1FQetfIpJE8ApdvdVxTg==", + "requires": { + "jwa": "^2.0.0", + "safe-buffer": "^5.0.1" + } + }, "levn": { "version": "0.3.0", "resolved": "https://registry.npmjs.org/levn/-/levn-0.3.0.tgz", @@ -4151,6 +4192,11 @@ "rimraf": "~2.4.0" } }, + "nan": { + "version": "2.14.2", + "resolved": "https://registry.npmjs.org/nan/-/nan-2.14.2.tgz", + "integrity": "sha512-M2ufzIiINKCuDfBSAUr1vWQ+vuVcA9kqx8JJUsbQi6yf1uGRyb7HfpdfUr5qLXf3B/t8dPvcjhKMmlfnP47EzQ==" + }, "nanoid": { "version": "3.1.20", "resolved": "https://registry.npmjs.org/nanoid/-/nanoid-3.1.20.tgz", @@ -4170,9 +4216,9 @@ "optional": true }, "needle": { - "version": "2.5.2", - "resolved": "https://registry.npmjs.org/needle/-/needle-2.5.2.tgz", - "integrity": "sha512-LbRIwS9BfkPvNwNHlsA41Q29kL2L/6VaOJ0qisM5lLWsTV3nP15abO5ITL6L81zqFhzjRKDAYjpcBcwM0AVvLQ==", + "version": "2.6.0", + "resolved": "https://registry.npmjs.org/needle/-/needle-2.6.0.tgz", + "integrity": "sha512-KKYdza4heMsEfSWD7VPUIz3zX2XDwOyX2d+geb4vrERZMT5RMU6ujjaD+I5Yr54uZxQ2w6XRTAhHBbSCyovZBg==", "requires": { "debug": "^3.2.6", "iconv-lite": "^0.4.4", @@ -4188,9 +4234,9 @@ } }, "ms": { - "version": "2.1.2", - "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.2.tgz", - "integrity": "sha512-sGkPx+VjMtmA6MX27oA4FBFELFCZZ4S4XqeGOXCv68tT+jb3vk/RyaKWP0PTKyWtmLSM0b+adUTEvbs1PEaH2w==" + "version": "2.1.3", + "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.3.tgz", + "integrity": "sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA==" } } }, @@ -4240,6 +4286,11 @@ "resolved": "https://registry.npmjs.org/node-fetch/-/node-fetch-2.6.1.tgz", "integrity": "sha512-V4aYg89jEoVRxRb2fJdAg8FHvI7cEyYdVAh94HH0UIK8oJxUfkjlDQN9RbMx+bEjP7+ggMiFRprSti032Oipxw==" }, + "node-forge": { + "version": "0.10.0", + "resolved": "https://registry.npmjs.org/node-forge/-/node-forge-0.10.0.tgz", + "integrity": "sha512-PPmu8eEeG9saEUvI97fm4OYxXVB6bFvyNTyiUOBichBpFG8A1Ljw3bY62+5oOjDEMHRnd0Y7HQ+x7uzxOzC6JA==" + }, "node-pre-gyp": { "version": "0.16.0", "resolved": "https://registry.npmjs.org/node-pre-gyp/-/node-pre-gyp-0.16.0.tgz", @@ -4536,6 +4587,11 @@ "callsites": "^3.0.0" } }, + "parse-duration": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/parse-duration/-/parse-duration-1.0.0.tgz", + "integrity": "sha512-X4kUkCTHU1N/kEbwK9FpUJ0UZQa90VzeczfS704frR30gljxDG0pSziws06XlK+CGRSo/1wtG1mFIdBFQTMQNw==" + }, "parse-json": { "version": "2.2.0", "resolved": "https://registry.npmjs.org/parse-json/-/parse-json-2.2.0.tgz", @@ -4548,7 +4604,7 @@ "parse-ms": { "version": "2.1.0", "resolved": "https://registry.npmjs.org/parse-ms/-/parse-ms-2.1.0.tgz", - "integrity": "sha1-NIVlp1PUOR+lJAKZVrFyy3dTCX0=" + "integrity": "sha512-kHt7kzLoS9VBZfUsiKjv43mr91ea+U05EyKkEtqp7vNbHxmaVuEqN7XxeEVnGrMtYOAxGrDElSi96K7EgO1zCA==" }, "parseurl": { "version": "1.3.3", @@ -4616,6 +4672,11 @@ "integrity": "sha512-q0M/9eZHzmr0AulXyPwNfZjtwZ/RBZlbN3K3CErVrk50T2ASYI7Bye0EvekFY3IP1Nt2DHu0re+V2ZHIpMkuWg==", "dev": true }, + "pify": { + "version": "5.0.0", + "resolved": "https://registry.npmjs.org/pify/-/pify-5.0.0.tgz", + "integrity": "sha512-eW/gHNMlxdSP6dmG6uJip6FXN0EQBwm2clYYd8Wul42Cwu/DK8HEftzsapcNdYe2MfLiIwZqsDk2RDEsTE79hA==" + }, "pkg-dir": { "version": "2.0.0", "resolved": "https://registry.npmjs.org/pkg-dir/-/pkg-dir-2.0.0.tgz", @@ -4647,10 +4708,10 @@ "split": "^1.0.1" }, "dependencies": { - "nan": { - "version": "2.14.2", - "resolved": "https://registry.npmjs.org/nan/-/nan-2.14.2.tgz", - "integrity": "sha512-M2ufzIiINKCuDfBSAUr1vWQ+vuVcA9kqx8JJUsbQi6yf1uGRyb7HfpdfUr5qLXf3B/t8dPvcjhKMmlfnP47EzQ==" + "delay": { + "version": "4.4.1", + "resolved": "https://registry.npmjs.org/delay/-/delay-4.4.1.tgz", + "integrity": "sha512-aL3AhqtfhOlT/3ai6sWXeqwnw63ATNpnUiN4HL7x9q+My5QtHlO3OIkasmug9LKzpheLdmUKGRKnYXYAS7FQkQ==" }, "p-limit": { "version": "3.1.0", @@ -4660,11 +4721,6 @@ "yocto-queue": "^0.1.0" } }, - "pify": { - "version": "5.0.0", - "resolved": "https://registry.npmjs.org/pify/-/pify-5.0.0.tgz", - "integrity": "sha512-eW/gHNMlxdSP6dmG6uJip6FXN0EQBwm2clYYd8Wul42Cwu/DK8HEftzsapcNdYe2MfLiIwZqsDk2RDEsTE79hA==" - }, "protobufjs": { "version": "6.10.2", "resolved": "https://registry.npmjs.org/protobufjs/-/protobufjs-6.10.2.tgz", @@ -5313,6 +5369,14 @@ } } }, + "pretty-ms": { + "version": "7.0.1", + "resolved": "https://registry.npmjs.org/pretty-ms/-/pretty-ms-7.0.1.tgz", + "integrity": "sha512-973driJZvxiGOQ5ONsFhOF/DtzPMOMtgC11kCpUrPGMTgqp2q/1gwzCquocrN33is0VZ5GFHXZYMM9l6h67v2Q==", + "requires": { + "parse-ms": "^2.1.0" + } + }, "process-nextick-args": { "version": "2.0.1", "resolved": "https://registry.npmjs.org/process-nextick-args/-/process-nextick-args-2.0.1.tgz", @@ -5654,6 +5718,31 @@ "integrity": "sha512-fGxEI7+wsG9xrvdjsrlmL22OMTTiHRwAMroiEeMgq8gzoLC/PQr7RsRDSTLUg/bZAZtF+TVIkHc6/4RIKrui+Q==", "dev": true }, + "require-in-the-middle": { + "version": "5.1.0", + "resolved": "https://registry.npmjs.org/require-in-the-middle/-/require-in-the-middle-5.1.0.tgz", + "integrity": "sha512-M2rLKVupQfJ5lf9OvqFGIT+9iVLnTmjgbOmpil12hiSQNn5zJTKGPoIisETNjfK+09vP3rpm1zJajmErpr2sEQ==", + "requires": { + "debug": "^4.1.1", + "module-details-from-path": "^1.0.3", + "resolve": "^1.12.0" + }, + "dependencies": { + "debug": { + "version": "4.3.1", + "resolved": "https://registry.npmjs.org/debug/-/debug-4.3.1.tgz", + "integrity": "sha512-doEwdvm4PCeK4K3RQN2ZC2BYUBaxwLARCqZmMjtF8a51J2Rb0xpVloFRnCODwqjpwnAoao4pelN8l3RJdv3gRQ==", + "requires": { + "ms": "2.1.2" + } + }, + "ms": { + "version": "2.1.2", + "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.2.tgz", + "integrity": "sha512-sGkPx+VjMtmA6MX27oA4FBFELFCZZ4S4XqeGOXCv68tT+jb3vk/RyaKWP0PTKyWtmLSM0b+adUTEvbs1PEaH2w==" + } + } + }, "require-like": { "version": "0.1.2", "resolved": "https://registry.npmjs.org/require-like/-/require-like-0.1.2.tgz", @@ -6235,6 +6324,25 @@ "bintrees": "1.0.1" } }, + "teeny-request": { + "version": "7.0.1", + "resolved": "https://registry.npmjs.org/teeny-request/-/teeny-request-7.0.1.tgz", + "integrity": "sha512-sasJmQ37klOlplL4Ia/786M5YlOcoLGQyq2TE4WHSRupbAuDaQW0PfVxV4MtdBtRJ4ngzS+1qim8zP6Zp35qCw==", + "requires": { + "http-proxy-agent": "^4.0.0", + "https-proxy-agent": "^5.0.0", + "node-fetch": "^2.6.1", + "stream-events": "^1.0.5", + "uuid": "^8.0.0" + }, + "dependencies": { + "uuid": { + "version": "8.3.2", + "resolved": "https://registry.npmjs.org/uuid/-/uuid-8.3.2.tgz", + "integrity": "sha512-+NYs2QeMWy+GWFOEm9xnn6HCDp0l7QBD7ml8zLUmJ+93Q5NF0NocErnwkTkXVFNiX3/fpC6afS8Dhb/gz7R7eg==" + } + } + }, "text-table": { "version": "0.2.0", "resolved": "https://registry.npmjs.org/text-table/-/text-table-0.2.0.tgz", diff --git a/services/real-time/package.json b/services/real-time/package.json index 51ee7a1a0e..1d36d12466 100644 --- a/services/real-time/package.json +++ b/services/real-time/package.json @@ -21,7 +21,7 @@ "format:fix": "node_modules/.bin/prettier-eslint $PWD'/**/*.js' --write" }, "dependencies": { - "@overleaf/metrics": "^3.4.1", + "@overleaf/metrics": "^3.5.1", "@overleaf/o-error": "^3.1.0", "@overleaf/redis-wrapper": "^2.0.0", "async": "^0.9.0", From a55aa61d715095685f4f64d9218b05384880b3aa Mon Sep 17 00:00:00 2001 From: Brian Gough Date: Mon, 29 Mar 2021 11:13:59 +0100 Subject: [PATCH 471/491] use .includes instead of .indexOf --- services/real-time/app/js/DeploymentManager.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/services/real-time/app/js/DeploymentManager.js b/services/real-time/app/js/DeploymentManager.js index a3201e6b21..ddb98fd4ce 100644 --- a/services/real-time/app/js/DeploymentManager.js +++ b/services/real-time/app/js/DeploymentManager.js @@ -12,7 +12,7 @@ const deploymentColour = settings.deploymentColour var serviceCloseTime function updateDeploymentStatus(fileContent) { - const closed = fileContent && fileContent.indexOf(deploymentColour) === -1 + const closed = fileContent && !fileContent.includes(deploymentColour) if (closed && !settings.serviceIsClosed) { settings.serviceIsClosed = true serviceCloseTime = Date.now() + 60 * 1000 // delay closing by 1 minute From e63b9ec5ea02ebf5ff7ee819511e6b8868b0ecf2 Mon Sep 17 00:00:00 2001 From: Thomas Mees Date: Tue, 30 Mar 2021 12:34:25 +0200 Subject: [PATCH 472/491] Lower log level of 'no project_id found on client' messages --- services/real-time/app/js/Router.js | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/services/real-time/app/js/Router.js b/services/real-time/app/js/Router.js index 9b93517ee4..9b677e39d8 100644 --- a/services/real-time/app/js/Router.js +++ b/services/real-time/app/js/Router.js @@ -44,12 +44,15 @@ module.exports = Router = { metrics.inc('unexpected-arguments', 1, { status: method }) const serializedError = { message: error.message } callback(serializedError) + } else if (error.message === 'no project_id found on client') { + logger.debug(attrs, error.message) + const serializedError = { message: error.message } + callback(serializedError) } else if ( [ 'not authorized', 'joinLeaveEpoch mismatch', - 'doc updater could not load requested ops', - 'no project_id found on client' + 'doc updater could not load requested ops' ].includes(error.message) ) { logger.warn(attrs, error.message) From 9c50bbffd1fe4aeb7cc6f44de5325ddd83144004 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 1 Apr 2021 03:49:36 +0000 Subject: [PATCH 473/491] Bump y18n from 4.0.0 to 4.0.1 Bumps [y18n](https://github.com/yargs/y18n) from 4.0.0 to 4.0.1. - [Release notes](https://github.com/yargs/y18n/releases) - [Changelog](https://github.com/yargs/y18n/blob/master/CHANGELOG.md) - [Commits](https://github.com/yargs/y18n/commits) Signed-off-by: dependabot[bot] --- services/real-time/package-lock.json | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/services/real-time/package-lock.json b/services/real-time/package-lock.json index 3a55a51434..7e3dbbedb6 100644 --- a/services/real-time/package-lock.json +++ b/services/real-time/package-lock.json @@ -4130,12 +4130,6 @@ "strip-ansi": "^6.0.0" } }, - "y18n": { - "version": "5.0.5", - "resolved": "https://registry.npmjs.org/y18n/-/y18n-5.0.5.tgz", - "integrity": "sha512-hsRUr4FFrvhhRH12wOdfs38Gy7k2FFzB9qgN9v3aLykRq0dRcdcpz5C9FxdS2NuhOrI/628b/KSTJ3rwHysYSg==", - "dev": true - }, "yargs": { "version": "16.2.0", "resolved": "https://registry.npmjs.org/yargs/-/yargs-16.2.0.tgz", @@ -6806,9 +6800,9 @@ "integrity": "sha1-AUU6HZvtHo8XL2SVu/TIxCYyFQA=" }, "y18n": { - "version": "4.0.0", - "resolved": "https://registry.npmjs.org/y18n/-/y18n-4.0.0.tgz", - "integrity": "sha512-r9S/ZyXu/Xu9q1tYlpsLIsa3EeLXXk0VwlxqTcFRfg9EhMW+17kbt9G0NrgCmhGb5vT2hyhJZLfDGx+7+5Uj/w==", + "version": "5.0.5", + "resolved": "https://registry.npmjs.org/y18n/-/y18n-5.0.5.tgz", + "integrity": "sha512-hsRUr4FFrvhhRH12wOdfs38Gy7k2FFzB9qgN9v3aLykRq0dRcdcpz5C9FxdS2NuhOrI/628b/KSTJ3rwHysYSg==", "dev": true }, "yallist": { @@ -6884,6 +6878,12 @@ "is-fullwidth-code-point": "^2.0.0", "strip-ansi": "^5.1.0" } + }, + "y18n": { + "version": "4.0.1", + "resolved": "https://registry.npmjs.org/y18n/-/y18n-4.0.1.tgz", + "integrity": "sha512-wNcy4NvjMYL8gogWWYAO7ZFWFfHcbdbE57tZO8e4cbpj8tfUcwrwqSl3ad8HxpYWCdXcJUCeKKZS62Av1affwQ==", + "dev": true } } }, From a6689f00f6074b518dfd2f02086a61b35c6aaf1a Mon Sep 17 00:00:00 2001 From: Jakob Ackermann Date: Thu, 29 Apr 2021 15:30:52 +0100 Subject: [PATCH 474/491] [misc] add linting for missing explicit dependencies and fix any errors --- services/real-time/.eslintrc | 13 +++++++++++-- services/real-time/buildscript.txt | 2 +- 2 files changed, 12 insertions(+), 3 deletions(-) diff --git a/services/real-time/.eslintrc b/services/real-time/.eslintrc index 76dad1561d..321353f971 100644 --- a/services/real-time/.eslintrc +++ b/services/real-time/.eslintrc @@ -22,7 +22,10 @@ "rules": { // Swap the no-unused-expressions rule with a more chai-friendly one "no-unused-expressions": 0, - "chai-friendly/no-unused-expressions": "error" + "chai-friendly/no-unused-expressions": "error", + + // Do not allow importing of implicit dependencies. + "import/no-extraneous-dependencies": "error" }, "overrides": [ { @@ -57,7 +60,13 @@ "files": ["app/**/*.js", "app.js", "index.js"], "rules": { // don't allow console.log in backend code - "no-console": "error" + "no-console": "error", + + // Do not allow importing of implicit dependencies. + "import/no-extraneous-dependencies": ["error", { + // Do not allow importing of devDependencies. + "devDependencies": false + }] } } ] diff --git a/services/real-time/buildscript.txt b/services/real-time/buildscript.txt index 109e42ca58..d8fc7d8c7d 100644 --- a/services/real-time/buildscript.txt +++ b/services/real-time/buildscript.txt @@ -5,4 +5,4 @@ real-time --env-pass-through= --node-version=12.21.0 --public-repo=True ---script-version=3.7.0 +--script-version=3.8.0 From 19507962db2229b3c29c2dccd84fdb52f88a10e1 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Fri, 7 May 2021 05:18:17 +0000 Subject: [PATCH 475/491] Bump underscore from 1.7.0 to 1.13.1 Bumps [underscore](https://github.com/jashkenas/underscore) from 1.7.0 to 1.13.1. - [Release notes](https://github.com/jashkenas/underscore/releases) - [Commits](https://github.com/jashkenas/underscore/compare/1.7.0...1.13.1) Signed-off-by: dependabot[bot] --- services/real-time/package-lock.json | 6 +++--- services/real-time/package.json | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/services/real-time/package-lock.json b/services/real-time/package-lock.json index 3a55a51434..f24a0eb4f9 100644 --- a/services/real-time/package-lock.json +++ b/services/real-time/package-lock.json @@ -6526,9 +6526,9 @@ } }, "underscore": { - "version": "1.7.0", - "resolved": "https://registry.npmjs.org/underscore/-/underscore-1.7.0.tgz", - "integrity": "sha1-a7rwh3UA02vjTsqlhODbn+8DUgk=" + "version": "1.13.1", + "resolved": "https://registry.npmjs.org/underscore/-/underscore-1.13.1.tgz", + "integrity": "sha512-hzSoAVtJF+3ZtiFX0VgfFPHEDRm7Y/QPjGyNo4TVdnDTdft3tr8hEkD25a1jC+TjTuE7tkHGKkhwCgs9dgBB2g==" }, "unpipe": { "version": "1.0.0", diff --git a/services/real-time/package.json b/services/real-time/package.json index 1d36d12466..0fd71cc74e 100644 --- a/services/real-time/package.json +++ b/services/real-time/package.json @@ -37,7 +37,7 @@ "settings-sharelatex": "^1.1.0", "socket.io": "https://github.com/overleaf/socket.io/archive/0.9.19-overleaf-4.tar.gz", "socket.io-client": "https://github.com/overleaf/socket.io-client/archive/0.9.17-overleaf-3.tar.gz", - "underscore": "1.7.0" + "underscore": "1.13.1" }, "devDependencies": { "bunyan": "~0.22.3", From 9622b341b75a2dd69423aeb92faa9a5b9df82833 Mon Sep 17 00:00:00 2001 From: Brian Gough Date: Mon, 10 May 2021 10:16:50 +0100 Subject: [PATCH 476/491] add transport type to metrics --- services/real-time/app/js/Router.js | 4 ++-- services/real-time/app/js/WebsocketController.js | 14 ++++++++------ 2 files changed, 10 insertions(+), 8 deletions(-) diff --git a/services/real-time/app/js/Router.js b/services/real-time/app/js/Router.js index 9b677e39d8..28765ecfa8 100644 --- a/services/real-time/app/js/Router.js +++ b/services/real-time/app/js/Router.js @@ -153,7 +153,7 @@ module.exports = Router = { client.publicId = 'P.' + base64id.generateId() client.emit('connectionAccepted', null, client.publicId) - metrics.inc('socket-io.connection') + metrics.inc('socket-io.connection', 1, { status: client.transport }) metrics.gauge('socket-io.clients', io.sockets.clients().length) logger.log({ session, client_id: client.id }, 'client connected') @@ -211,7 +211,7 @@ module.exports = Router = { }) client.on('disconnect', function () { - metrics.inc('socket-io.disconnect') + metrics.inc('socket-io.disconnect', 1, { status: client.transport }) metrics.gauge('socket-io.clients', io.sockets.clients().length) WebsocketController.leaveProject(io, client, function (err) { diff --git a/services/real-time/app/js/WebsocketController.js b/services/real-time/app/js/WebsocketController.js index f8e32f0788..a54291b0c7 100644 --- a/services/real-time/app/js/WebsocketController.js +++ b/services/real-time/app/js/WebsocketController.js @@ -36,7 +36,7 @@ module.exports = WebsocketController = { { user_id, project_id, client_id: client.id }, 'user joining project' ) - metrics.inc('editor.join-project') + metrics.inc('editor.join-project', 1, { status: client.transport }) WebApiManager.joinProject(project_id, user, function ( error, project, @@ -114,7 +114,7 @@ module.exports = WebsocketController = { return callback() } // client did not join project - metrics.inc('editor.leave-project') + metrics.inc('editor.leave-project', 1, { status: client.transport }) logger.log( { project_id, user_id, client_id: client.id }, 'client leaving project' @@ -167,7 +167,7 @@ module.exports = WebsocketController = { } const joinLeaveEpoch = ++client.joinLeaveEpoch - metrics.inc('editor.join-doc') + metrics.inc('editor.join-doc', 1, { status: client.transport }) const { project_id, user_id, is_restricted_user } = client.ol_context if (!project_id) { return callback(new NotJoinedError()) @@ -319,7 +319,7 @@ module.exports = WebsocketController = { leaveDoc(client, doc_id, callback) { // client may have disconnected, but we have to cleanup internal state. client.joinLeaveEpoch++ - metrics.inc('editor.leave-doc') + metrics.inc('editor.leave-doc', 1, { status: client.transport }) const { project_id, user_id } = client.ol_context logger.log( { user_id, project_id, doc_id, client_id: client.id }, @@ -338,7 +338,9 @@ module.exports = WebsocketController = { return callback() } - metrics.inc('editor.update-client-position', 0.1) + metrics.inc('editor.update-client-position', 0.1, { + status: client.transport + }) const { project_id, first_name, @@ -475,7 +477,7 @@ module.exports = WebsocketController = { } update.meta.source = client.publicId update.meta.user_id = user_id - metrics.inc('editor.doc-update', 0.3) + metrics.inc('editor.doc-update', 0.3, { status: client.transport }) logger.log( { From d5f2f4b7e4a552e508ba2da0db54cb5a97b55bd1 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 11 May 2021 03:16:58 +0000 Subject: [PATCH 477/491] Bump lodash from 4.17.20 to 4.17.21 Bumps [lodash](https://github.com/lodash/lodash) from 4.17.20 to 4.17.21. - [Release notes](https://github.com/lodash/lodash/releases) - [Commits](https://github.com/lodash/lodash/compare/4.17.20...4.17.21) Signed-off-by: dependabot[bot] --- services/real-time/package-lock.json | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/services/real-time/package-lock.json b/services/real-time/package-lock.json index 3a55a51434..3ba00b850a 100644 --- a/services/real-time/package-lock.json +++ b/services/real-time/package-lock.json @@ -3522,9 +3522,9 @@ } }, "lodash": { - "version": "4.17.20", - "resolved": "https://registry.npmjs.org/lodash/-/lodash-4.17.20.tgz", - "integrity": "sha512-PlhdFcillOINfeV7Ni6oF1TAEayyZBoZ8bcshTHqOYJYlrqzRK5hagpagky5o4HfCzzd1TRkXPMFq6cKk9rGmA==", + "version": "4.17.21", + "resolved": "https://registry.npmjs.org/lodash/-/lodash-4.17.21.tgz", + "integrity": "sha512-v2kDEe57lecTulaDIuNTPy3Ry4gLGJ6Z1O3vE1krgXZNrsQ+LFTGHVxVjcXPs17LhbZVGedAJv8XZ1tvj5FvSg==", "dev": true }, "lodash.at": { From 8a16c55000ed75b8dd7f2b4eaa4fea5e5e731e69 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 11 May 2021 20:43:06 +0000 Subject: [PATCH 478/491] Bump hosted-git-info from 2.8.8 to 2.8.9 Bumps [hosted-git-info](https://github.com/npm/hosted-git-info) from 2.8.8 to 2.8.9. - [Release notes](https://github.com/npm/hosted-git-info/releases) - [Changelog](https://github.com/npm/hosted-git-info/blob/v2.8.9/CHANGELOG.md) - [Commits](https://github.com/npm/hosted-git-info/compare/v2.8.8...v2.8.9) Signed-off-by: dependabot[bot] --- services/real-time/package-lock.json | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/services/real-time/package-lock.json b/services/real-time/package-lock.json index 3a55a51434..aadfdc0b2c 100644 --- a/services/real-time/package-lock.json +++ b/services/real-time/package-lock.json @@ -2988,9 +2988,9 @@ "integrity": "sha512-Yu+q/XWr2fFQ11tHxPq4p4EiNkb2y+lAacJNhAdRXVfRIcDH6gi7htWFnnlIzvqHMHoWeIsfXlNAjZInpAOJDA==" }, "hosted-git-info": { - "version": "2.8.8", - "resolved": "https://registry.npmjs.org/hosted-git-info/-/hosted-git-info-2.8.8.tgz", - "integrity": "sha512-f/wzC2QaWBs7t9IYqB4T3sR1xviIViXJRJTWBlx2Gf3g0Xi5vI7Yy4koXQ1c9OYDGHN9sBy1DQ2AB8fqZBWhUg==", + "version": "2.8.9", + "resolved": "https://registry.npmjs.org/hosted-git-info/-/hosted-git-info-2.8.9.tgz", + "integrity": "sha512-mxIDAb9Lsm6DoOJ7xH+5+X4y1LU/4Hi50L9C5sIswK3JzULS4bwk1FvjdBgvYR4bzT4tuUQiC15FE2f5HbLvYw==", "dev": true }, "http-errors": { From 3044bc4cf5c36d222f0135c8fad8f089bd8b782c Mon Sep 17 00:00:00 2001 From: Brian Gough Date: Thu, 13 May 2021 14:51:26 +0100 Subject: [PATCH 479/491] add transport to get-connected-users metric --- services/real-time/app/js/WebsocketController.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/services/real-time/app/js/WebsocketController.js b/services/real-time/app/js/WebsocketController.js index a54291b0c7..bdf27d6967 100644 --- a/services/real-time/app/js/WebsocketController.js +++ b/services/real-time/app/js/WebsocketController.js @@ -414,7 +414,7 @@ module.exports = WebsocketController = { return callback() } - metrics.inc('editor.get-connected-users') + metrics.inc('editor.get-connected-users', { status: client.transport }) const { project_id, user_id, is_restricted_user } = client.ol_context if (is_restricted_user) { return callback(null, []) From 219d6832df367a6eb6732ac31c0af9e5fc6b6f5d Mon Sep 17 00:00:00 2001 From: Brian Gough Date: Wed, 19 May 2021 14:28:54 +0100 Subject: [PATCH 480/491] upgrade to socket.io 0.9.19-overleaf-5 fix for exception in client.transport getter --- services/real-time/package-lock.json | 4 ++-- services/real-time/package.json | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/services/real-time/package-lock.json b/services/real-time/package-lock.json index 6fd1ba98f8..2126ce30d2 100644 --- a/services/real-time/package-lock.json +++ b/services/real-time/package-lock.json @@ -6028,8 +6028,8 @@ } }, "socket.io": { - "version": "https://github.com/overleaf/socket.io/archive/0.9.19-overleaf-4.tar.gz", - "integrity": "sha512-jWgdvVEbPioarWhfKvmtFf9miv/TYMePwrWO+r3WlVF+07nPkWY+OK0y0Lob5shC/dTLqwyG9ajw49+ObC8s/A==", + "version": "https://github.com/overleaf/socket.io/archive/0.9.19-overleaf-5.tar.gz", + "integrity": "sha512-MDRh05EWE7OSgLzsFR0ikLzIVxPD7ItC5FcScxY58QYTRmC4p0kbod4zVSYjIT9aTdMM6CnWXMvFYSe50vV/iA==", "requires": { "base64id": "0.1.0", "policyfile": "0.0.4", diff --git a/services/real-time/package.json b/services/real-time/package.json index 0fd71cc74e..27d44c64bb 100644 --- a/services/real-time/package.json +++ b/services/real-time/package.json @@ -35,7 +35,7 @@ "logger-sharelatex": "^2.2.0", "request": "^2.88.2", "settings-sharelatex": "^1.1.0", - "socket.io": "https://github.com/overleaf/socket.io/archive/0.9.19-overleaf-4.tar.gz", + "socket.io": "https://github.com/overleaf/socket.io/archive/0.9.19-overleaf-5.tar.gz", "socket.io-client": "https://github.com/overleaf/socket.io-client/archive/0.9.17-overleaf-3.tar.gz", "underscore": "1.13.1" }, From 1160e9e98f72aca1778422d5ba94b46a4ae59979 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Sat, 12 Jun 2021 00:00:31 +0000 Subject: [PATCH 481/491] Bump glob-parent from 5.1.1 to 5.1.2 Bumps [glob-parent](https://github.com/gulpjs/glob-parent) from 5.1.1 to 5.1.2. - [Release notes](https://github.com/gulpjs/glob-parent/releases) - [Changelog](https://github.com/gulpjs/glob-parent/blob/main/CHANGELOG.md) - [Commits](https://github.com/gulpjs/glob-parent/compare/v5.1.1...v5.1.2) --- updated-dependencies: - dependency-name: glob-parent dependency-type: indirect ... Signed-off-by: dependabot[bot] --- services/real-time/package-lock.json | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/services/real-time/package-lock.json b/services/real-time/package-lock.json index 2126ce30d2..92ae478ba4 100644 --- a/services/real-time/package-lock.json +++ b/services/real-time/package-lock.json @@ -2702,9 +2702,9 @@ } }, "glob-parent": { - "version": "5.1.1", - "resolved": "https://registry.npmjs.org/glob-parent/-/glob-parent-5.1.1.tgz", - "integrity": "sha512-FnI+VGOpnlGHWZxthPGR+QhR78fuiK0sNLkHQv+bL9fQi57lNNdquIbna/WrfROrolq8GK5Ek6BiMwqL/voRYQ==", + "version": "5.1.2", + "resolved": "https://registry.npmjs.org/glob-parent/-/glob-parent-5.1.2.tgz", + "integrity": "sha512-AOIgSQCepiJYwP3ARnGx+5VnTu2HBYdzbGP45eLw1vr3zB3vZLeyed1sC9hnbcOc9/SrMyM5RPQrkGz4aS9Zow==", "dev": true, "requires": { "is-glob": "^4.0.1" From 7af2c6a5ea5f6db937098221ae3db037f7e33552 Mon Sep 17 00:00:00 2001 From: Jakob Ackermann Date: Mon, 12 Jul 2021 16:58:05 +0100 Subject: [PATCH 482/491] [misc] install bunyan as production dependency ``` Error: Cannot find module 'bunyan' Require stack: - .../node_modules/@google-cloud/logging-bunyan/build/src/middleware/express.js - .../node_modules/@google-cloud/logging-bunyan/build/src/index.js - .../node_modules/logger-sharelatex/logging-manager.js - .../node_modules/logger-sharelatex/index.js - .../app.js ``` --- services/real-time/package-lock.json | 232 +++++++++++++-------------- services/real-time/package.json | 2 +- 2 files changed, 112 insertions(+), 122 deletions(-) diff --git a/services/real-time/package-lock.json b/services/real-time/package-lock.json index 2126ce30d2..ba777f5d4c 100644 --- a/services/real-time/package-lock.json +++ b/services/real-time/package-lock.json @@ -706,7 +706,7 @@ "@protobufjs/aspromise": { "version": "1.1.2", "resolved": "https://registry.npmjs.org/@protobufjs/aspromise/-/aspromise-1.1.2.tgz", - "integrity": "sha1-m4sMxmPWaafY9vXQiToU00jzD78=" + "integrity": "sha512-j+gKExEuLmKwvz3OgROXtrJ2UG2x8Ch2YZUxahh+s1F2HZ+wAceUNLkvy6zKCPVRkU++ZWQrdxsUeQXmcg4uoQ==" }, "@protobufjs/base64": { "version": "1.1.2", @@ -721,12 +721,12 @@ "@protobufjs/eventemitter": { "version": "1.1.0", "resolved": "https://registry.npmjs.org/@protobufjs/eventemitter/-/eventemitter-1.1.0.tgz", - "integrity": "sha1-NVy8mLr61ZePntCV85diHx0Ga3A=" + "integrity": "sha512-j9ednRT81vYJ9OfVuXG6ERSTdEL1xVsNgqpkxMsbIabzSo3goCjDIveeGv5d03om39ML71RdmrGNjG5SReBP/Q==" }, "@protobufjs/fetch": { "version": "1.1.0", "resolved": "https://registry.npmjs.org/@protobufjs/fetch/-/fetch-1.1.0.tgz", - "integrity": "sha1-upn7WYYUr2VwDBYZ/wbUVLDYTEU=", + "integrity": "sha512-lljVXpqXebpsijW71PZaCYeIcE5on1w5DlQy5WH6GLbFryLUrBD4932W/E2BSpfRJWseIL4v/KPgBFxDOIdKpQ==", "requires": { "@protobufjs/aspromise": "^1.1.1", "@protobufjs/inquire": "^1.1.0" @@ -735,27 +735,27 @@ "@protobufjs/float": { "version": "1.0.2", "resolved": "https://registry.npmjs.org/@protobufjs/float/-/float-1.0.2.tgz", - "integrity": "sha1-Xp4avctz/Ap8uLKR33jIy9l7h9E=" + "integrity": "sha512-Ddb+kVXlXst9d+R9PfTIxh1EdNkgoRe5tOX6t01f1lYWOvJnSPDBlG241QLzcyPdoNTsblLUdujGSE4RzrTZGQ==" }, "@protobufjs/inquire": { "version": "1.1.0", "resolved": "https://registry.npmjs.org/@protobufjs/inquire/-/inquire-1.1.0.tgz", - "integrity": "sha1-/yAOPnzyQp4tyvwRQIKOjMY48Ik=" + "integrity": "sha512-kdSefcPdruJiFMVSbn801t4vFK7KB/5gd2fYvrxhuJYg8ILrmn9SKSX2tZdV6V+ksulWqS7aXjBcRXl3wHoD9Q==" }, "@protobufjs/path": { "version": "1.1.2", "resolved": "https://registry.npmjs.org/@protobufjs/path/-/path-1.1.2.tgz", - "integrity": "sha1-bMKyDFya1q0NzP0hynZz2Nf79o0=" + "integrity": "sha512-6JOcJ5Tm08dOHAbdR3GrvP+yUUfkjG5ePsHYczMFLq3ZmMkAD98cDgcT2iA1lJ9NVwFd4tH/iSSoe44YWkltEA==" }, "@protobufjs/pool": { "version": "1.1.0", "resolved": "https://registry.npmjs.org/@protobufjs/pool/-/pool-1.1.0.tgz", - "integrity": "sha1-Cf0V8tbTq/qbZbw2ZQbWrXhG/1Q=" + "integrity": "sha512-0kELaGSIDBKvcgS4zkjz1PeddatrjYcmMWOlAuAPwAeccUrPHdUqo/J6LiymHHEiJT5NrF1UVwxY14f+fy4WQw==" }, "@protobufjs/utf8": { "version": "1.1.0", "resolved": "https://registry.npmjs.org/@protobufjs/utf8/-/utf8-1.1.0.tgz", - "integrity": "sha1-p3c2C1s5oaLlEG+OhY8v0tBgxXA=" + "integrity": "sha512-Vvn3zZrhQZkkBE8LSuW3em98c0FwgO4nxzv6OdSxPKJIEKY2bGbHn+mhGIPerzI4twdxaP8/0+06HBpwf345Lw==" }, "@sinonjs/commons": { "version": "1.8.2", @@ -932,7 +932,7 @@ "abort-controller": { "version": "3.0.0", "resolved": "https://registry.npmjs.org/abort-controller/-/abort-controller-3.0.0.tgz", - "integrity": "sha1-6vVNU7YrrkE46AnKIlyEOabvs5I=", + "integrity": "sha512-h8lQ8tacZYnR3vNQTgibj+tODHI5/+l06Au2Pcriv/Gmet0eaj4TwWH41sO9wnHDiQsEj19q0drzdWdeAHtweg==", "requires": { "event-target-shim": "^5.0.0" } @@ -1089,7 +1089,7 @@ "array-flatten": { "version": "1.1.1", "resolved": "https://registry.npmjs.org/array-flatten/-/array-flatten-1.1.1.tgz", - "integrity": "sha1-ml9pkFGx5wczKPKgCJaLZOopVdI=" + "integrity": "sha512-PCVAQswWemu6UdxsDFFX/+gVeYqKAod3D3UVm91jHwynguOwAvYPhx8nNlM++NqRcK6CxxpUafjmhIdKiHibqg==" }, "array-includes": { "version": "3.1.1", @@ -1115,7 +1115,7 @@ "arrify": { "version": "2.0.1", "resolved": "https://registry.npmjs.org/arrify/-/arrify-2.0.1.tgz", - "integrity": "sha1-yWVekzHgq81YjSp8rX6ZVvZnAfo=" + "integrity": "sha512-3duEwti880xqi4eAMN8AyR4a0ByT90zoYdLlevfrvU43vb0YZwZVfxOgxWrLXXXpyugL0hNZc9G6BiB5B3nUug==" }, "asn1": { "version": "0.2.4", @@ -1128,12 +1128,12 @@ "assert-plus": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/assert-plus/-/assert-plus-1.0.0.tgz", - "integrity": "sha1-8S4PPF13sLHN2RRpQuTpbB5N1SU=" + "integrity": "sha512-NfJ4UzBCcQGLDlQq7nHxH+tv3kyZ0hHQqF5BO6J7tNJeP5do1llPr8dZ8zHonfhAu0PHAdMkSo+8o0wxg9lZWw==" }, "assertion-error": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/assertion-error/-/assertion-error-1.0.0.tgz", - "integrity": "sha1-x/hUOP3UZrx8oWq5DIFRN5el0js=", + "integrity": "sha512-g/gZV+G476cnmtYI+Ko9d5khxSoCSoom/EaNmmCfwpOvBXEJ18qwFrxfP1/CsIqk2no1sAKKwxndV0tP7ROOFQ==", "dev": true }, "astral-regex": { @@ -1145,7 +1145,7 @@ "async": { "version": "0.9.2", "resolved": "https://registry.npmjs.org/async/-/async-0.9.2.tgz", - "integrity": "sha1-rqdNXmHB+JlhO/ZL2mbUx48v0X0=" + "integrity": "sha512-l6ToIJIotphWahxxHyzK9bnLR6kM4jJIIgLShZeqLY7iboHoGkdgFl7W2/Ivi4SkMJYGKqW8vSuk0uKUj6qsSw==" }, "async-listener": { "version": "0.6.10", @@ -1166,12 +1166,12 @@ "asynckit": { "version": "0.4.0", "resolved": "https://registry.npmjs.org/asynckit/-/asynckit-0.4.0.tgz", - "integrity": "sha1-x57Zf380y48robyXkLzDZkdLS3k=" + "integrity": "sha512-Oei9OH4tRh0YqU3GxhX79dM/mwVgvbZJaSNaRk+bshkj0S5cfHcgYakreBjrHwatXKbz+IoIdYLxrKim2MjW0Q==" }, "aws-sign2": { "version": "0.7.0", "resolved": "https://registry.npmjs.org/aws-sign2/-/aws-sign2-0.7.0.tgz", - "integrity": "sha1-tG6JCTSpWR8tL2+G1+ap8bP+dqg=" + "integrity": "sha512-08kcGqnYf/YmjoRhfxyu+CLxBjUtHLXLXX/vUfx9l2LYzG3c1m61nrpyFUZI6zeS+Li/wWMMidD9KgrqtGq3mA==" }, "aws4": { "version": "1.10.0", @@ -1181,7 +1181,7 @@ "balanced-match": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/balanced-match/-/balanced-match-1.0.0.tgz", - "integrity": "sha1-ibTRmasr7kneFk6gK4nORi1xt2c=" + "integrity": "sha512-9Y0g0Q8rmSt+H33DfKv7FOc3v+iRI+o1lbzt8jGcIosYW37IIW/2XVYq5NPdmaD5NQ59Nk26Kl/vZbwW9Fr8vg==" }, "base64-js": { "version": "1.3.1", @@ -1191,17 +1191,17 @@ "base64id": { "version": "0.1.0", "resolved": "https://registry.npmjs.org/base64id/-/base64id-0.1.0.tgz", - "integrity": "sha1-As4P3u4M709ACA4ec+g08LG/zj8=" + "integrity": "sha512-DSjtfjhAsHl9J4OJj7e4+toV2zqxJrGwVd3CLlsCp8QmicvOn7irG0Mb8brOc/nur3SdO8lIbNlY1s1ZDJdUKQ==" }, "basic-auth-connect": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/basic-auth-connect/-/basic-auth-connect-1.0.0.tgz", - "integrity": "sha1-/bC0OWLKe0BFanwrtI/hc9otISI=" + "integrity": "sha512-kiV+/DTgVro4aZifY/hwRwALBISViL5NP4aReaR2EVJEObpbUBHIkdJh/YpcoEiYt7nBodZ6U2ajZeZvSxUCCg==" }, "bcrypt-pbkdf": { "version": "1.0.2", "resolved": "https://registry.npmjs.org/bcrypt-pbkdf/-/bcrypt-pbkdf-1.0.2.tgz", - "integrity": "sha1-pDAdOJtqQ/m2f/PKEaP2Y342Dp4=", + "integrity": "sha512-qeFIXtP4MSoi6NLqO12WfqARWWuCKi2Rn/9hJLEmtB5yTNr9DqFWkJRCf2qShWzPeAMRnOgCrq0sg/KLv5ES9w==", "requires": { "tweetnacl": "^0.14.3" } @@ -1280,7 +1280,7 @@ "buffer-equal-constant-time": { "version": "1.0.1", "resolved": "https://registry.npmjs.org/buffer-equal-constant-time/-/buffer-equal-constant-time-1.0.1.tgz", - "integrity": "sha1-+OcRMvf/5uAaXJaXpMbz5I1cyBk=" + "integrity": "sha512-zRpUiDwd/xk6ADqPMATG8vc9VPrkck7T07OIx0gnjmJAnHnTVXNQG3vfvWNuiZIkwu9KrKdA1iJKfsfTVxE6NA==" }, "buffer-from": { "version": "1.1.1", @@ -1293,13 +1293,14 @@ "integrity": "sha512-lGzLKcioL90C7wMczpkY0n/oART3MbBa8R9OFGE1rJxoVI86u4WAGfEk8Wjv10eKSyTHVGkSo3bvBylCEtk7LA==" }, "bunyan": { - "version": "0.22.3", - "resolved": "https://registry.npmjs.org/bunyan/-/bunyan-0.22.3.tgz", - "integrity": "sha1-ehncG0yMZF90AkGnQPIkUUfGfsI=", - "dev": true, + "version": "1.8.15", + "resolved": "https://registry.npmjs.org/bunyan/-/bunyan-1.8.15.tgz", + "integrity": "sha512-0tECWShh6wUysgucJcBAoYegf3JJoZWibxdqhTm7OHPeT42qdjkZ29QCMcKwbgU1kiH+auSIasNRXMLWXafXig==", "requires": { - "dtrace-provider": "0.2.8", - "mv": "~2" + "dtrace-provider": "~0.8", + "moment": "^2.19.3", + "mv": "~2", + "safe-json-stringify": "~1" } }, "bytes": { @@ -1333,12 +1334,12 @@ "caseless": { "version": "0.12.0", "resolved": "https://registry.npmjs.org/caseless/-/caseless-0.12.0.tgz", - "integrity": "sha1-G2gcIf+EAzyCZUMJBolCDRhxUdw=" + "integrity": "sha512-4tYFyifaFfGacoiObjJegolkwSU4xQNGbVgUiNYVUxbQ2x2lUsFvY4hVgVzGiIe6WLOPqycWXA40l+PWsxthUw==" }, "chai": { "version": "1.9.2", "resolved": "https://registry.npmjs.org/chai/-/chai-1.9.2.tgz", - "integrity": "sha1-Pxog+CsLnXQ3V30k1vErGmnTtZA=", + "integrity": "sha512-olRoaitftnzWHFEAza6MXR4w+FfZrOVyV7r7U/Z8ObJefCgL8IuWkAuASJjSXrpP9wvgoL8+1dB9RbMLc2FkNg==", "dev": true, "requires": { "assertion-error": "1.0.0", @@ -1442,7 +1443,7 @@ "cluster-key-slot": { "version": "1.1.0", "resolved": "https://registry.npmjs.org/cluster-key-slot/-/cluster-key-slot-1.1.0.tgz", - "integrity": "sha1-MEdLKpgfsSFyaVgzBSvA0BM20Q0=" + "integrity": "sha512-2Nii8p3RwAPiFwsnZvukotvow2rIHM+yQ6ZcBXGHdniadkYGZYiGmkHJIbZPIV9nfv7m/U1IPMVVcAhoWFeklw==" }, "code-point-at": { "version": "1.1.0", @@ -1532,12 +1533,12 @@ "concat-map": { "version": "0.0.1", "resolved": "https://registry.npmjs.org/concat-map/-/concat-map-0.0.1.tgz", - "integrity": "sha1-2Klr13/Wjfd5OnMDajug1UBdR3s=" + "integrity": "sha512-/Srv4dswyQNBfohGpz9o6Yb3Gz3SrUDqBH5rTuhGR7ahtlbYKnVxw2bCFMRljaA7EXHaXZ8wsHdodFvbkhKmqg==" }, "connect-redis": { "version": "2.5.1", "resolved": "https://registry.npmjs.org/connect-redis/-/connect-redis-2.5.1.tgz", - "integrity": "sha1-6MCF227Gg7T8RXzzP5PD5+1vA/c=", + "integrity": "sha512-mnPCfN12SYcTw1yYRLejwv6WMmWpNtMkLeVIPzzAkC+u/bY8GxvMezG7wUKpMNoLmggS5xG2DWjEelggv6s5cw==", "requires": { "debug": "^1.0.4", "redis": "^0.12.1" @@ -1546,7 +1547,7 @@ "debug": { "version": "1.0.5", "resolved": "https://registry.npmjs.org/debug/-/debug-1.0.5.tgz", - "integrity": "sha1-9yQSF0MPmd7EwrRz6rkiKOh0wqw=", + "integrity": "sha512-SIKSrp4+XqcUaNWhwaPJbLFnvSXPsZ4xBdH2WRK0Xo++UzMC4eepYghGAVhVhOwmfq3kqowqJ5w45R3pmYZnuA==", "requires": { "ms": "2.0.0" } @@ -1639,7 +1640,7 @@ "core-util-is": { "version": "1.0.2", "resolved": "https://registry.npmjs.org/core-util-is/-/core-util-is-1.0.2.tgz", - "integrity": "sha1-tf1UIgqivFq1eqtxQMlAdUUDwac=" + "integrity": "sha512-3lqz5YjWTYnW6dlDa5TLaTCcShfar1e40rmcJVwCBJC6mWlFuj0eCHIElmG1g5kyuJ/GD+8Wn4FFCcz4gJPfaQ==" }, "cross-spawn": { "version": "6.0.5", @@ -1675,7 +1676,7 @@ "dashdash": { "version": "1.14.1", "resolved": "https://registry.npmjs.org/dashdash/-/dashdash-1.14.1.tgz", - "integrity": "sha1-hTz6D3y+L+1d4gMmuN1YEDX24vA=", + "integrity": "sha512-jRFi8UDGo6j+odZiEpjazZaWqEal3w/basFjQHQEwVtZJGDpxbH1MeYluwCS8Xq5wmLJooDlMgvVarmWfGM44g==", "requires": { "assert-plus": "^1.0.0" } @@ -1697,7 +1698,7 @@ "deep-eql": { "version": "0.1.3", "resolved": "https://registry.npmjs.org/deep-eql/-/deep-eql-0.1.3.tgz", - "integrity": "sha1-71WKyrjeJSBs1xOQbXTlaTDrafI=", + "integrity": "sha512-6sEotTRGBFiNcqVoeHwnfopbSpi5NbH1VWJmYCVkmxMmaVTT0bUTrNaGyBwhgP4MZL012W/mkzIn3Da+iDYweg==", "dev": true, "requires": { "type-detect": "0.1.1" @@ -1731,7 +1732,7 @@ "delayed-stream": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/delayed-stream/-/delayed-stream-1.0.0.tgz", - "integrity": "sha1-3zrhmayt+31ECqrgsp4icrJOxhk=" + "integrity": "sha512-ZySD7Nf91aLB0RxL4KGrKHBXl7Eds1DAmEdcoVawXnLD7SDhpNgtuII2aAkg7a7QS41jxPSZ17p4VdGnMHk3MQ==" }, "delegates": { "version": "1.0.0", @@ -1746,12 +1747,12 @@ "depd": { "version": "1.1.2", "resolved": "https://registry.npmjs.org/depd/-/depd-1.1.2.tgz", - "integrity": "sha1-m81S4UwJd2PnSbJ0xDRu0uVgtak=" + "integrity": "sha512-7emPTl6Dpo6JRXOXjLRxck+FlLRX5847cLKEn00PLAgc3g2hTZZgr+e4c2v6QpSmLeFP3n5yUo7ft6avBK/5jQ==" }, "destroy": { "version": "1.0.4", "resolved": "https://registry.npmjs.org/destroy/-/destroy-1.0.4.tgz", - "integrity": "sha1-l4hXRCxEdJ5CBmE+N5RiBYJqvYA=" + "integrity": "sha512-3NdhDuEXnfun/z7x9GOElY49LoqVHoGScmOKwmxhsS8N5Y+Z8KyPPDnaSzqWgYt/ji4mqwfTS34Htrk0zPIXVg==" }, "detect-libc": { "version": "1.0.3", @@ -1788,16 +1789,18 @@ } }, "dtrace-provider": { - "version": "0.2.8", - "resolved": "https://registry.npmjs.org/dtrace-provider/-/dtrace-provider-0.2.8.tgz", - "integrity": "sha1-4kPxkhmqlfvw2PL/sH9b1k6U/iA=", - "dev": true, - "optional": true + "version": "0.8.8", + "resolved": "https://registry.npmjs.org/dtrace-provider/-/dtrace-provider-0.8.8.tgz", + "integrity": "sha512-b7Z7cNtHPhH9EJhNNbbeqTcXB8LGFFZhq1PGgEvpeHlzd36bhbdTWoE/Ba/YguqpBSlAPKnARWhVlhunCMwfxg==", + "optional": true, + "requires": { + "nan": "^2.14.0" + } }, "duplexify": { "version": "3.7.1", "resolved": "https://registry.npmjs.org/duplexify/-/duplexify-3.7.1.tgz", - "integrity": "sha1-Kk31MX9sz9kfhtb9JdjYoQO4gwk=", + "integrity": "sha512-07z8uv2wMyS51kKhD1KsdXJg5WQ6t93RneqRxUHnskXVtlYYkLqM0gqStQZ3pj073g687jPCHrqNfCzawLYh5g==", "requires": { "end-of-stream": "^1.0.0", "inherits": "^2.0.1", @@ -1808,7 +1811,7 @@ "ecc-jsbn": { "version": "0.1.2", "resolved": "https://registry.npmjs.org/ecc-jsbn/-/ecc-jsbn-0.1.2.tgz", - "integrity": "sha1-OoOpBOVDUyh4dMVkt1SThoSamMk=", + "integrity": "sha512-eh9O+hwRHNbG4BLTjEl3nw044CkGm5X6LoaCf7LPp7UU8Qrt47JYNi6nPX8xjW97TKGKm1ouctg0QSpZe9qrnw==", "requires": { "jsbn": "~0.1.0", "safer-buffer": "^2.1.0" @@ -1817,7 +1820,7 @@ "ecdsa-sig-formatter": { "version": "1.0.11", "resolved": "https://registry.npmjs.org/ecdsa-sig-formatter/-/ecdsa-sig-formatter-1.0.11.tgz", - "integrity": "sha1-rg8PothQRe8UqBfao86azQSJ5b8=", + "integrity": "sha512-nagl3RYrbNv6kQkeJIpt6NJZy8twLB/2vtz6yN9Z4vRKHN4/QZJIEbqohALSgwKdnksuY3k5Addp5lg8sVoVcQ==", "requires": { "safe-buffer": "^5.0.1" } @@ -1825,7 +1828,7 @@ "ee-first": { "version": "1.1.1", "resolved": "https://registry.npmjs.org/ee-first/-/ee-first-1.1.1.tgz", - "integrity": "sha1-WQxhFWsK4vTwJVcyoViyZrxWsh0=" + "integrity": "sha512-WMwm9LhRUo+WUaRN+vRuETqG89IgZphVSNkdFgeb6sS/E4OrDIN7t48CAewSHXc6C8lefD8KKfr5vY61brQlow==" }, "emitter-listener": { "version": "1.1.2", @@ -1844,7 +1847,7 @@ "encodeurl": { "version": "1.0.2", "resolved": "https://registry.npmjs.org/encodeurl/-/encodeurl-1.0.2.tgz", - "integrity": "sha1-rT/0yG7C0CkyL1oCw6mmBslbP1k=" + "integrity": "sha512-TPJXq8JqFaVYm2CWmPvnP2Iyo4ZSM7/QKcSmuMLDObfpH5fi7RUGmd/rTDf+rut/saiDiQEeVTNgAmJEdAOx0w==" }, "end-of-stream": { "version": "1.4.4", @@ -1857,7 +1860,7 @@ "ent": { "version": "2.2.0", "resolved": "https://registry.npmjs.org/ent/-/ent-2.2.0.tgz", - "integrity": "sha1-6WQhkyWiHQX0RGai9obtbOX13R0=" + "integrity": "sha512-GHrMyVZQWvTIdDtpiEXdHZnFQKzeO09apj8Cbl4pKWy4i0Oprcq17usfDt5aO63swf0JOeMWjWQE/LzgSRuWpA==" }, "error-ex": { "version": "1.3.2", @@ -1907,12 +1910,12 @@ "escape-html": { "version": "1.0.3", "resolved": "https://registry.npmjs.org/escape-html/-/escape-html-1.0.3.tgz", - "integrity": "sha1-Aljq5NPQwJdN4cFpGI7wBR0dGYg=" + "integrity": "sha512-NiSupZ4OeuGwr68lGIeym/ksIZMJodUGOSCZ/FSnTxcrekbvqrgdUxlJOMpijaKZVjAJrWrGs/6Jy8OMuyj9ow==" }, "escape-string-regexp": { "version": "1.0.5", "resolved": "https://registry.npmjs.org/escape-string-regexp/-/escape-string-regexp-1.0.5.tgz", - "integrity": "sha1-G2HAViGQqN/2rjuyzwIAyhMLhtQ=", + "integrity": "sha512-vbRorB5FUQWvla16U8R/qgaFIya2qGzwDrNmCZuYKrbdSUMG6I1ZCGQRefkRVhuOkIGVne7BQ35DSfo1qvJqFg==", "dev": true }, "eslint": { @@ -2257,12 +2260,12 @@ "etag": { "version": "1.8.1", "resolved": "https://registry.npmjs.org/etag/-/etag-1.8.1.tgz", - "integrity": "sha1-Qa4u62XvpiJorr/qg6x9eSmbCIc=" + "integrity": "sha512-aIL5Fx7mawVa300al2BnEE4iNvo1qETxLrPI/o05L7z6go7fCw1J6EQmbK4FmJ2AS7kgVF/KEZWufBfdClMcPg==" }, "event-target-shim": { "version": "5.0.1", "resolved": "https://registry.npmjs.org/event-target-shim/-/event-target-shim-5.0.1.tgz", - "integrity": "sha1-XU0+vflYPWOlMzzi3rdICrKwV4k=" + "integrity": "sha512-i/2XbnSz/uxRCU6+NdVJgKWDTM427+MqYbkQzD321DuCQJUqOuJKIA0IM2+W2xtYHdKOmZ4dR6fExsd4SXL+WQ==" }, "eventid": { "version": "1.0.0", @@ -2390,7 +2393,7 @@ "extsprintf": { "version": "1.3.0", "resolved": "https://registry.npmjs.org/extsprintf/-/extsprintf-1.3.0.tgz", - "integrity": "sha1-lpGEQOMEGnpBT4xS48V06zw+HgU=" + "integrity": "sha512-11Ndz7Nv+mvAC1j0ktTa7fAb0vLyGGX+rMHNBYQviQDGU0Hw7lhctJANqbPhu9nV9/izT/IntTgZ7Im/9LJs9g==" }, "fast-deep-equal": { "version": "3.1.3", @@ -2530,7 +2533,7 @@ "forever-agent": { "version": "0.6.1", "resolved": "https://registry.npmjs.org/forever-agent/-/forever-agent-0.6.1.tgz", - "integrity": "sha1-+8cfDEGt6zf5bFd60e1C2P2sypE=" + "integrity": "sha512-j0KLYPhm6zeac4lz3oJ3o65qvgQCcPubiyotZrXqEaG4hNagNYO8qdlUrX5vwqv9ohqeT/Z3j6+yW067yWWdUw==" }, "form-data": { "version": "2.3.3", @@ -2545,12 +2548,12 @@ "forwarded": { "version": "0.1.2", "resolved": "https://registry.npmjs.org/forwarded/-/forwarded-0.1.2.tgz", - "integrity": "sha1-mMI9qxF1ZXuMBXPozszZGw/xjIQ=" + "integrity": "sha512-Ua9xNhH0b8pwE3yRbFfXJvfdWF0UHNCdeyb2sbi9Ul/M+r3PTdrz7Cv4SCfZRMjmzEM9PhraqfZFbGTIg3OMyA==" }, "fresh": { "version": "0.5.2", "resolved": "https://registry.npmjs.org/fresh/-/fresh-0.5.2.tgz", - "integrity": "sha1-PYyt2Q2XZWn6g1qx+OSyOhBWBac=" + "integrity": "sha512-zJ2mQYM18rEFOudeV4GShTGIQ7RbzA7ozbU9I/XBpm7kqgMywgmylMwXHxZJmkVoYkna9d2pVXVXPdYTP9ej8Q==" }, "fs-minipass": { "version": "1.2.7", @@ -2563,7 +2566,7 @@ "fs.realpath": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/fs.realpath/-/fs.realpath-1.0.0.tgz", - "integrity": "sha1-FQStJSMVjKpA20onh8sBQRmU6k8=" + "integrity": "sha512-OO0pH2lK6a0hZnAdau5ItzHPI6pUlvI7jMVnxUQRtw4owF2wk8lOSabtGDCTP4Ggrg2MbGnWO9X8K1t4+fGMDw==" }, "fsevents": { "version": "2.3.2", @@ -2683,7 +2686,7 @@ "getpass": { "version": "0.1.7", "resolved": "https://registry.npmjs.org/getpass/-/getpass-0.1.7.tgz", - "integrity": "sha1-Xv+OPmhNVprkyysSgmBOi6YhSfo=", + "integrity": "sha512-0fzj9JxOLfJ+XGLhR8ze3unN0KZCgZwiSSDz168VERjK8Wl8kVSdcu2kspd4s4wtAa1y/qrVRiAA0WclVsu0ng==", "requires": { "assert-plus": "^1.0.0" } @@ -2922,7 +2925,7 @@ "har-schema": { "version": "2.0.0", "resolved": "https://registry.npmjs.org/har-schema/-/har-schema-2.0.0.tgz", - "integrity": "sha1-qUwiJOvKwEeCoNkDVSHyRzW37JI=" + "integrity": "sha512-Oqluz6zhGX8cyRaTQlFMPw80bSJVG2x/cFb8ZPhUILGgHka9SsokCCOQgpveePerqidZOrT14ipqfJb7ILcW5Q==" }, "har-validator": { "version": "5.1.3", @@ -3041,7 +3044,7 @@ "http-signature": { "version": "1.2.0", "resolved": "https://registry.npmjs.org/http-signature/-/http-signature-1.2.0.tgz", - "integrity": "sha1-muzZJRFHcvPZW2WmCruPfBj7rOE=", + "integrity": "sha512-CAbnr6Rz4CYQkLYUtSNXxQPUH2gK8f3iWexVlsnMeD+GjlsQ0Xsy1cOX+mN3dtxYomRy21CiOzU8Uhw6OwncEQ==", "requires": { "assert-plus": "^1.0.0", "jsprim": "^1.2.2", @@ -3119,7 +3122,7 @@ "inflight": { "version": "1.0.6", "resolved": "https://registry.npmjs.org/inflight/-/inflight-1.0.6.tgz", - "integrity": "sha1-Sb1jMdfQLQwJvJEKEHW6gWW1bfk=", + "integrity": "sha512-k92I/b08q4wvFscXCLvqfsHCrjrF7yiXsQuIVvVE7N82W3+aqpzuUdBbfhWcy/FZR3/4IgflMgKLOsvPDrGCJA==", "requires": { "once": "^1.3.0", "wrappy": "1" @@ -3128,7 +3131,7 @@ "inherits": { "version": "2.0.3", "resolved": "https://registry.npmjs.org/inherits/-/inherits-2.0.3.tgz", - "integrity": "sha1-Yzwsg+PaQqUC9SRmAiSA9CCCYd4=" + "integrity": "sha512-x00IRNXNy63jwGkJmzPigoySHbaqpNuzKbBOmzK+g2OdZpQ9w+sxCN+VSB3ja7IAge2OP2qpfxTjeNcyjmW1uw==" }, "ini": { "version": "1.3.8", @@ -3360,12 +3363,12 @@ "is-typedarray": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/is-typedarray/-/is-typedarray-1.0.0.tgz", - "integrity": "sha1-5HnICFjfDBsR3dppQPlgEfzaSpo=" + "integrity": "sha512-cyA56iCMHAh5CdzjJIa4aohJyeO1YbwLi3Jc35MmRU6poroFjIGZzUzupGiRPOjgHg9TLu43xbpwXk523fMxKA==" }, "isarray": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/isarray/-/isarray-1.0.0.tgz", - "integrity": "sha1-u5NdSFgsuhaMBoNJV6VKPgcSTxE=" + "integrity": "sha512-VLghIWNM6ELQzo7zwmcg0NmTVyWKYjvIeM83yjp0wRDTmUnrM678fQbcKBo6n2CJEF0szoG//ytg+TKla89ALQ==" }, "isexe": { "version": "2.0.0", @@ -3376,7 +3379,7 @@ "isstream": { "version": "0.1.2", "resolved": "https://registry.npmjs.org/isstream/-/isstream-0.1.2.tgz", - "integrity": "sha1-R+Y/evVa+m+S4VAOaQ64uFKcCZo=" + "integrity": "sha512-Yljz7ffyPbrLpLngrMtZ7NduUgVvi6wG9RJ9IUcyCd59YQ911PBJphODUcbOVbqYfxe1wuYf/LJ8PauMRwsM/g==" }, "js-tokens": { "version": "4.0.0", @@ -3397,12 +3400,12 @@ "jsbn": { "version": "0.1.1", "resolved": "https://registry.npmjs.org/jsbn/-/jsbn-0.1.1.tgz", - "integrity": "sha1-peZUwuWi3rXyAdls77yoDA7y9RM=" + "integrity": "sha512-UVU9dibq2JcFWxQPA6KCqj5O42VOmAY3zQUfEKxU0KpTGXwNoCjkX1e13eHNvw/xPynt6pU0rZ1htjWTNTSXsg==" }, "json-bigint": { "version": "0.3.0", "resolved": "https://registry.npmjs.org/json-bigint/-/json-bigint-0.3.0.tgz", - "integrity": "sha1-DM2RLEuCcNBfBW+9E4FLU9OCWx4=", + "integrity": "sha512-u+c/u/F+JNPUekHCFyGVycRPyh9UHD5iUhSyIAn10kxbDTJxijwAbT6XHaONEOXuGGfmWUSroheXgHcml4gLgg==", "requires": { "bignumber.js": "^7.0.0" } @@ -3410,7 +3413,7 @@ "json-schema": { "version": "0.2.3", "resolved": "https://registry.npmjs.org/json-schema/-/json-schema-0.2.3.tgz", - "integrity": "sha1-tIDIkuWaLwWVTOcnvT8qTogvnhM=" + "integrity": "sha512-a3xHnILGMtk+hDOqNwHzF6e2fNbiMrXZvxKQiEv2MlgQP+pjIOzqAmKYD2mDpXYE/44M7g+n9p2bKkYWDUcXCQ==" }, "json-schema-traverse": { "version": "0.4.1", @@ -3426,7 +3429,7 @@ "json-stringify-safe": { "version": "5.0.1", "resolved": "https://registry.npmjs.org/json-stringify-safe/-/json-stringify-safe-5.0.1.tgz", - "integrity": "sha1-Epai1Y/UXxmg9s4B1lcB4sc1tus=" + "integrity": "sha512-ZClg6AaYvamvYEE82d3Iyd3vSSIjQ+odgjaTzRuO3s7toCdFKczob2i0zCh7JE8kWn17yvAWhUVxvqGwUalsRA==" }, "json5": { "version": "1.0.1", @@ -3448,7 +3451,7 @@ "jsprim": { "version": "1.4.1", "resolved": "https://registry.npmjs.org/jsprim/-/jsprim-1.4.1.tgz", - "integrity": "sha1-MT5mvB5cwG5Di8G3SZwuXFastqI=", + "integrity": "sha512-4Dj8Rf+fQ+/Pn7C5qeEX02op1WfOss3PKTE9Nsop3Dx+6UPxlm1dr/og7o2cRa5hNN07CACr4NFzRLtj/rjWog==", "requires": { "assert-plus": "1.0.0", "extsprintf": "1.3.0", @@ -3654,22 +3657,10 @@ "yn": "^4.0.0" }, "dependencies": { - "bunyan": { - "version": "1.8.14", - "resolved": "https://registry.npmjs.org/bunyan/-/bunyan-1.8.14.tgz", - "integrity": "sha512-LlahJUxXzZLuw/hetUQJmRgZ1LF6+cr5TPpRj6jf327AsiIq2jhYEH4oqUUkVKTor+9w2BT3oxVwhzE5lw9tcg==", - "requires": { - "dtrace-provider": "~0.8", - "moment": "^2.19.3", - "mv": "~2", - "safe-json-stringify": "~1" - } - }, "dtrace-provider": { "version": "0.8.8", "resolved": "https://registry.npmjs.org/dtrace-provider/-/dtrace-provider-0.8.8.tgz", "integrity": "sha512-b7Z7cNtHPhH9EJhNNbbeqTcXB8LGFFZhq1PGgEvpeHlzd36bhbdTWoE/Ba/YguqpBSlAPKnARWhVlhunCMwfxg==", - "optional": true, "requires": { "nan": "^2.14.0" } @@ -3677,8 +3668,7 @@ "nan": { "version": "2.14.1", "resolved": "https://registry.npmjs.org/nan/-/nan-2.14.1.tgz", - "integrity": "sha512-isWHgVjnFjh2x2yuJ/tj3JbwoHu3UC2dX5G/88Cm24yB6YopVgxvBObDY7n5xW6ExmFhJpSEQqFPvq9zaXc8Jw==", - "optional": true + "integrity": "sha512-isWHgVjnFjh2x2yuJ/tj3JbwoHu3UC2dX5G/88Cm24yB6YopVgxvBObDY7n5xW6ExmFhJpSEQqFPvq9zaXc8Jw==" }, "yn": { "version": "4.0.0", @@ -3801,12 +3791,12 @@ "media-typer": { "version": "0.3.0", "resolved": "https://registry.npmjs.org/media-typer/-/media-typer-0.3.0.tgz", - "integrity": "sha1-hxDXrwqmJvj/+hzgAWhUUmMlV0g=" + "integrity": "sha512-dq+qelQ9akHpcOl/gUVRTxVIOkAJ1wR3QAvb4RsVjS8oVoFjDGTc679wJYmUmknUF5HwMLOgb5O+a3KxfWapPQ==" }, "merge-descriptors": { "version": "1.0.1", "resolved": "https://registry.npmjs.org/merge-descriptors/-/merge-descriptors-1.0.1.tgz", - "integrity": "sha1-sAqqVW3YtEVoFQ7J0blT8/kMu2E=" + "integrity": "sha512-cCi6g3/Zr1iqQi6ySbseM1Xvooa98N0w31jzUYrXPX2xqObmFGHJ0tQ5u74H3mVh7wLouTseZyYIq39g8cNp1w==" }, "messageformat": { "version": "2.3.0", @@ -3834,7 +3824,7 @@ "methods": { "version": "1.1.2", "resolved": "https://registry.npmjs.org/methods/-/methods-1.1.2.tgz", - "integrity": "sha1-VSmk1nZUE07cxSZmVoNbD4Ua/O4=" + "integrity": "sha512-iclAHeNqNm68zFtnZ0e+1L2yUIdvzNoauKU4WBA3VvH/vPFieF7qfRlwUZU+DA9P9bPXIS90ulxoUoCH23sV2w==" }, "mime": { "version": "1.6.0", @@ -3871,7 +3861,7 @@ "minimist": { "version": "0.0.8", "resolved": "https://registry.npmjs.org/minimist/-/minimist-0.0.8.tgz", - "integrity": "sha1-hX/Kv8M5fSYluCKCYuhqp6ARsF0=" + "integrity": "sha512-miQKw5Hv4NS1Psg2517mV4e4dYNaO3++hjAvLOAzKqZ61rH8NS1SK+vbfBWZ5PY/Me/bEWhUwqMghEW5Fb9T7Q==" }, "minipass": { "version": "2.9.0", @@ -3900,7 +3890,7 @@ "mkdirp": { "version": "0.5.1", "resolved": "https://registry.npmjs.org/mkdirp/-/mkdirp-0.5.1.tgz", - "integrity": "sha1-MAV0OOrGz3+MR2fzhkjWaX11yQM=", + "integrity": "sha512-SknJC52obPfGQPnjIkXbmA6+5H15E+fR+E4iR2oQ3zzCLbd7/ONua69R/Gw7AgkTLsRG+r5fzksYwWe1AgTyWA==", "requires": { "minimist": "0.0.8" } @@ -4159,15 +4149,15 @@ "integrity": "sha1-EUyUlnPiqKNenTV4hSeqN7Z52is=" }, "moment": { - "version": "2.27.0", - "resolved": "https://registry.npmjs.org/moment/-/moment-2.27.0.tgz", - "integrity": "sha512-al0MUK7cpIcglMv3YF13qSgdAIqxHTO7brRtaz3DlSULbqfazqkc5kEjNrLDOM7fsjshoFIihnU8snrP7zUvhQ==", + "version": "2.29.1", + "resolved": "https://registry.npmjs.org/moment/-/moment-2.29.1.tgz", + "integrity": "sha512-kHmoybcPV8Sqy59DwNDY3Jefr64lK/by/da0ViFcuA4DH0vQg5Q6Ze5VimxkfQNSC+Mls/Kx53s7TjP1RhFEDQ==", "optional": true }, "ms": { "version": "2.0.0", "resolved": "https://registry.npmjs.org/ms/-/ms-2.0.0.tgz", - "integrity": "sha1-VgiurfwAvmwpAd9fmGF4jeDVl8g=" + "integrity": "sha512-Tpp60P6IUJDTuOq/5Z8cdskzJujfwqfOTkrwIwj7IRISpnkJnT6SyJ4PCPnGMoFjC9ddhal5KVIYtAt97ix05A==" }, "mute-stream": { "version": "0.0.8", @@ -4466,7 +4456,7 @@ "on-finished": { "version": "2.3.0", "resolved": "https://registry.npmjs.org/on-finished/-/on-finished-2.3.0.tgz", - "integrity": "sha1-IPEzZIGwg811M3mSoWlxqi2QaUc=", + "integrity": "sha512-ikqdkGAAyf/X/gPhXGvfgAytDZtDbr+bkNUJ0N9h5MI/dmdgCs3l6hoHrcUv41sRKew3jIwrp4qQDXiK99Utww==", "requires": { "ee-first": "1.1.1" } @@ -4479,7 +4469,7 @@ "once": { "version": "1.4.0", "resolved": "https://registry.npmjs.org/once/-/once-1.4.0.tgz", - "integrity": "sha1-WDsap3WWHUsROsF9nFC6753Xa9E=", + "integrity": "sha512-lNaJgI+2Q5URQBkccEKHTQOPaXdUxnZZElQTZY0MFUAuaEqe1E+Nyvgdz/aIyNi6Z9MzO5dv1H8n58/GELp3+w==", "requires": { "wrappy": "1" } @@ -4569,7 +4559,7 @@ "p-try": { "version": "2.2.0", "resolved": "https://registry.npmjs.org/p-try/-/p-try-2.2.0.tgz", - "integrity": "sha1-yyhoVA4xPWHeWPr741zpAE1VQOY=", + "integrity": "sha512-R4nPAVTAU0B9D35/Gk3uJf/7XYbQcyohSKdvAxIRSNghFl4e71hVoGnBNQz9cWaXxO2I10KTC+3jMdvvoKw6dQ==", "dev": true }, "parent-module": { @@ -4614,7 +4604,7 @@ "path-is-absolute": { "version": "1.0.1", "resolved": "https://registry.npmjs.org/path-is-absolute/-/path-is-absolute-1.0.1.tgz", - "integrity": "sha1-F0uSaHNVNP+8es5r9TpanhtcX18=" + "integrity": "sha512-AVbw3UJ2e9bq64vSaS9Am0fje1Pa8pbGqTTsmXfaIiMpnr5DlDhfJOuLj9Sf95ZPVDAUerDfEk88MPmPe7UCQg==" }, "path-is-inside": { "version": "1.0.2", @@ -4636,7 +4626,7 @@ "path-to-regexp": { "version": "0.1.7", "resolved": "https://registry.npmjs.org/path-to-regexp/-/path-to-regexp-0.1.7.tgz", - "integrity": "sha1-32BBeABfUi8V60SQ5yR6G/qmf4w=" + "integrity": "sha512-5DFkuoqlv1uYQKxy8omFBeJPQcdoE07Kv2sferDCrAq1ohOU+MSDswDIbnx3YAM60qIOnYa53wBhXW0EbMonrQ==" }, "path-type": { "version": "2.0.0", @@ -4658,7 +4648,7 @@ "performance-now": { "version": "2.1.0", "resolved": "https://registry.npmjs.org/performance-now/-/performance-now-2.1.0.tgz", - "integrity": "sha1-Ywn04OX6kT7BxpMHrjZLSzd8nns=" + "integrity": "sha512-7EAHlyLHI56VEIdK57uwHdHKIaAGbnXPiw0yWbarQZOKaKpvUIgW0jWRVLiatnM+XXlSwsanIBH/hzGMJulMow==" }, "picomatch": { "version": "2.2.2", @@ -5498,7 +5488,7 @@ "random-bytes": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/random-bytes/-/random-bytes-1.0.0.tgz", - "integrity": "sha1-T2ih3Arli9P7lYSMMDJNt11kNgs=" + "integrity": "sha512-iv7LhNVO047HzYR3InF6pUcUsPQiHTM1Qal51DcGSuZFBil1aBBWG5eHPNek7bvILMaYJ/8RU1e8w1AMdHmLQQ==" }, "randombytes": { "version": "2.1.0", @@ -5614,12 +5604,12 @@ "redis": { "version": "0.12.1", "resolved": "https://registry.npmjs.org/redis/-/redis-0.12.1.tgz", - "integrity": "sha1-ZN92rQ/IrOuuvSoGReikj6xJGF4=" + "integrity": "sha512-DtqxdmgmVAO7aEyxaXBiUTvhQPOYznTIvmPzs9AwWZqZywM50JlFxQjFhicI+LVbaun7uwfO3izuvc1L8NlPKQ==" }, "redis-commands": { "version": "1.5.0", "resolved": "https://registry.npmjs.org/redis-commands/-/redis-commands-1.5.0.tgz", - "integrity": "sha1-gNLiBpj+aI8icSf/nlFkp90X54U=" + "integrity": "sha512-6KxamqpZ468MeQC3bkWmCB1fp56XL64D4Kf0zJSwDZbVLLm7KFkoIcHrgRvQ+sk8dnhySs7+yBg94yIkAK7aJg==" }, "redis-errors": { "version": "1.2.0", @@ -5740,7 +5730,7 @@ "require-like": { "version": "0.1.2", "resolved": "https://registry.npmjs.org/require-like/-/require-like-0.1.2.tgz", - "integrity": "sha1-rW8wwTvs15cBDEaK+ndcDAprR/o=", + "integrity": "sha512-oyrU88skkMtDdauHDuKVrgR+zuItqr6/c//FXzvmxRGMexSDc6hNvJInGW3LL46n+8b50RykrvwSUIIQH2LQ5A==", "dev": true }, "require-main-filename": { @@ -5846,7 +5836,7 @@ "sandboxed-module": { "version": "0.3.0", "resolved": "https://registry.npmjs.org/sandboxed-module/-/sandboxed-module-0.3.0.tgz", - "integrity": "sha1-8fvvvYCaT2kHO9B8rm/H2y6vX2o=", + "integrity": "sha512-/2IfB1wtca3eNVPXbQrb6UkhE/1pV4Wz+5CdG6DPYqeaDsYDzxglBT7/cVaqyrlRyQKdmw+uTZUTRos9FFD2PQ==", "dev": true, "requires": { "require-like": "0.1.2", @@ -5856,7 +5846,7 @@ "stack-trace": { "version": "0.0.6", "resolved": "https://registry.npmjs.org/stack-trace/-/stack-trace-0.0.6.tgz", - "integrity": "sha1-HnGb1qJin/CcGJ4Xqe+QKpT8XbA=", + "integrity": "sha512-5/6uZt7RYjjAl8z2j1mXWAewz+I4Hk2/L/3n6NRLIQ31+uQ7nMd9O6G69QCdrrufHv0QGRRHl/jwUEGTqhelTA==", "dev": true } } @@ -5939,7 +5929,7 @@ "coffee-script": { "version": "1.6.0", "resolved": "https://registry.npmjs.org/coffee-script/-/coffee-script-1.6.0.tgz", - "integrity": "sha1-gIs5bhEPU9AhoZpO8fZb4OjjX6M=" + "integrity": "sha512-Tx8itEfCsQp8RbLDFt7qwjqXycAx2g6SI7//4PPUR2j6meLmNifYm6zKrNDcU1+Q/GWRhjhEZk7DaLG1TfIzGA==" } } }, @@ -6139,12 +6129,12 @@ "standard-as-callback": { "version": "2.0.1", "resolved": "https://registry.npmjs.org/standard-as-callback/-/standard-as-callback-2.0.1.tgz", - "integrity": "sha1-7YuyVkjhWDF1m2Ajvbh+a2CzgSY=" + "integrity": "sha512-NQOxSeB8gOI5WjSaxjBgog2QFw55FV8TkS6Y07BiB3VJ8xNTvUYm0wl0s8ObgQ5NhdpnNfigMIKjgPESzgr4tg==" }, "statuses": { "version": "1.5.0", "resolved": "https://registry.npmjs.org/statuses/-/statuses-1.5.0.tgz", - "integrity": "sha1-Fhx9rBd2Wf2YEfQ3cfqZOBR4Yow=" + "integrity": "sha512-OpZ3zP+jT1PI7I8nemJX4AKmAX070ZkYPVWV/AaKTJl+tXCTGyVdC1a4SL8RUQYEwk/f34ZX8UTykN68FwrqAA==" }, "stream-events": { "version": "1.0.5", @@ -6346,7 +6336,7 @@ "through": { "version": "2.3.8", "resolved": "https://registry.npmjs.org/through/-/through-2.3.8.tgz", - "integrity": "sha1-DdTJ/6q8NXlgsbckEV1+Doai4fU=" + "integrity": "sha512-w89qg7PI8wAdvX60bMDP+bFoD5Dvhm9oLheFp5O4a2QF0cSBGsBX4qZmadPMvVqlLJBBci+WqGGOAPvcDeNSVg==" }, "through2": { "version": "3.0.1", @@ -6364,7 +6354,7 @@ "timekeeper": { "version": "0.0.4", "resolved": "https://registry.npmjs.org/timekeeper/-/timekeeper-0.0.4.tgz", - "integrity": "sha1-kNt58X2Ni1NiFUOJSSuXJ2LP0nY=", + "integrity": "sha512-QSNovcsPIbrI9zzXTesL/iiDrS+4IT+0xCxFzDSI2/yHkHVL1QEB5FhzrKMzCEwsbSOISEsH1yPUZ6/Fve9DZQ==", "dev": true }, "tinycolor": { @@ -6445,7 +6435,7 @@ "tunnel-agent": { "version": "0.6.0", "resolved": "https://registry.npmjs.org/tunnel-agent/-/tunnel-agent-0.6.0.tgz", - "integrity": "sha1-J6XeoGs2sEoKmWZ3SykIaPD8QP0=", + "integrity": "sha512-McnNiV1l8RYeY8tBgEpuodCC1mLUdbSN+CYBL7kJsJNInOP8UjDDEwdk6Mw60vdLLrr5NHKZhMAOSrR2NZuQ+w==", "requires": { "safe-buffer": "^5.0.1" } @@ -6453,7 +6443,7 @@ "tweetnacl": { "version": "0.14.5", "resolved": "https://registry.npmjs.org/tweetnacl/-/tweetnacl-0.14.5.tgz", - "integrity": "sha1-WuaBd/GS1EViadEIr6k/+HQ/T2Q=" + "integrity": "sha512-KXXFFdAbFXY4geFIwoyNK+f5Z1b7swfXABfL7HXCmoIWMKU3dmS26672A4EeQtDzLKy7SXmfBu51JolvEKwtGA==" }, "type-check": { "version": "0.3.2", @@ -6467,7 +6457,7 @@ "type-detect": { "version": "0.1.1", "resolved": "https://registry.npmjs.org/type-detect/-/type-detect-0.1.1.tgz", - "integrity": "sha1-C6XsKohWQORw6k6FBZcZANrFiCI=", + "integrity": "sha512-5rqszGVwYgBoDkIm2oUtvkfZMQ0vk29iDMU0W2qCa3rG0vPDNczCMT4hV/bLBgLg8k8ri6+u3Zbt+S/14eMzlA==", "dev": true }, "type-fest": { @@ -6527,7 +6517,7 @@ "unpipe": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/unpipe/-/unpipe-1.0.0.tgz", - "integrity": "sha1-sr9O6FFKrmFltIF4KdIbLvSZBOw=" + "integrity": "sha512-pjy2bYhSsufwWlKwPc+l3cN7+wuJlK6uz0YdJEOlQDbl6jo/YlPi4mb8agUkVC8BF7V8NuzeyPNqRksA3hztKQ==" }, "uri-js": { "version": "4.2.2", @@ -6540,12 +6530,12 @@ "util-deprecate": { "version": "1.0.2", "resolved": "https://registry.npmjs.org/util-deprecate/-/util-deprecate-1.0.2.tgz", - "integrity": "sha1-RQ1Nyfpw3nMnYvvS1KKJgUGaDM8=" + "integrity": "sha512-EPD5q1uXyFxJpCrLnCc1nHnq3gOa6DZBocAIiI2TaSCA7VCJ1UJDMagCzIkXNsUYfD1daK//LTEQ8xiIbrHtcw==" }, "utils-merge": { "version": "1.0.1", "resolved": "https://registry.npmjs.org/utils-merge/-/utils-merge-1.0.1.tgz", - "integrity": "sha1-n5VxD1CiZ5R7LMwSR0HBAoQn5xM=" + "integrity": "sha512-pMZTvIkT1d+TFGvDOqodOclx0QWkkgi6Tdoa8gC8ffGAAqz9pzPTZWAybbsHHoED/ztMtkv/VoYTYyShUn81hA==" }, "uuid": { "version": "7.0.3", @@ -6571,12 +6561,12 @@ "vary": { "version": "1.1.2", "resolved": "https://registry.npmjs.org/vary/-/vary-1.1.2.tgz", - "integrity": "sha1-IpnwLG3tMNSllhsLn3RSShj2NPw=" + "integrity": "sha512-BNGbWLfd0eUPabhkXUVm0j8uuvREyTh5ovRa/dyow/BqAbZJyC+5fU+IzQOzmAKzYqYRAISoRhdQr3eIZ/PXqg==" }, "verror": { "version": "1.10.0", "resolved": "https://registry.npmjs.org/verror/-/verror-1.10.0.tgz", - "integrity": "sha1-OhBcoXBTr1XW4nDB+CiGguGNpAA=", + "integrity": "sha512-ZZKSmDAEFOijERBLkmYfJ+vmk3w+7hOLYDNkRCuRuMJGEmqYNCNLyBBFwWKVMhfwaEF3WOd0Zlw86U/WC/+nYw==", "requires": { "assert-plus": "^1.0.0", "core-util-is": "1.0.2", @@ -6765,7 +6755,7 @@ "wrappy": { "version": "1.0.2", "resolved": "https://registry.npmjs.org/wrappy/-/wrappy-1.0.2.tgz", - "integrity": "sha1-tSQ9jz7BqjXxNkYFvA0QNuMKtp8=" + "integrity": "sha512-l4Sp/DRseor9wL6EvV2+TuQn63dMkPjZ/sp9XkghTEbV9KlPS1xUsZ3u7/IQO4wxtcFB4bgpQPRcR3QCvezPcQ==" }, "write": { "version": "1.0.3", diff --git a/services/real-time/package.json b/services/real-time/package.json index 27d44c64bb..f07ab3da62 100644 --- a/services/real-time/package.json +++ b/services/real-time/package.json @@ -28,6 +28,7 @@ "base64id": "0.1.0", "basic-auth-connect": "^1.0.0", "body-parser": "^1.19.0", + "bunyan": "^1.8.15", "connect-redis": "^2.1.0", "cookie-parser": "^1.4.5", "express": "^4.17.1", @@ -40,7 +41,6 @@ "underscore": "1.13.1" }, "devDependencies": { - "bunyan": "~0.22.3", "chai": "~1.9.1", "cookie-signature": "^1.1.0", "eslint": "^6.8.0", From a26ae735974bfb0666455f5fe7ffc84a3b707756 Mon Sep 17 00:00:00 2001 From: Jakob Ackermann Date: Mon, 12 Jul 2021 17:47:18 +0100 Subject: [PATCH 483/491] [misc] switch from settings-sharelatex to @overleaf/settings --- services/real-time/app.js | 2 +- services/real-time/app/js/ChannelManager.js | 2 +- .../real-time/app/js/ConnectedUsersManager.js | 2 +- .../real-time/app/js/DeploymentManager.js | 2 +- .../app/js/DocumentUpdaterController.js | 2 +- .../app/js/DocumentUpdaterManager.js | 2 +- services/real-time/app/js/EventLogger.js | 2 +- services/real-time/app/js/Router.js | 2 +- services/real-time/app/js/SafeJsonParse.js | 2 +- services/real-time/app/js/WebApiManager.js | 2 +- .../real-time/app/js/WebsocketLoadBalancer.js | 2 +- services/real-time/package-lock.json | 20 +++++-------------- services/real-time/package.json | 2 +- .../test/acceptance/js/ApplyUpdateTests.js | 2 +- .../test/acceptance/js/DrainManagerTests.js | 2 +- .../test/acceptance/js/EarlyDisconnect.js | 2 +- .../test/acceptance/js/LeaveProjectTests.js | 2 +- .../test/acceptance/js/MatrixTests.js | 2 +- .../test/acceptance/js/PubSubRace.js | 2 +- .../test/acceptance/js/ReceiveUpdateTests.js | 2 +- .../test/acceptance/js/SessionSocketsTests.js | 2 +- .../acceptance/js/helpers/RealTimeClient.js | 2 +- .../acceptance/js/helpers/RealtimeServer.js | 2 +- .../test/unit/js/ChannelManagerTests.js | 2 +- .../unit/js/ConnectedUsersManagerTests.js | 2 +- .../unit/js/DocumentUpdaterControllerTests.js | 2 +- .../unit/js/DocumentUpdaterManagerTests.js | 2 +- .../test/unit/js/RoomManagerTests.js | 2 +- .../test/unit/js/SafeJsonParseTest.js | 2 +- .../test/unit/js/WebApiManagerTests.js | 2 +- 30 files changed, 34 insertions(+), 44 deletions(-) diff --git a/services/real-time/app.js b/services/real-time/app.js index bde00b2e17..0b8be4d17b 100644 --- a/services/real-time/app.js +++ b/services/real-time/app.js @@ -1,5 +1,5 @@ const Metrics = require('@overleaf/metrics') -const Settings = require('settings-sharelatex') +const Settings = require('@overleaf/settings') Metrics.initialize(Settings.appName || 'real-time') const async = require('async') diff --git a/services/real-time/app/js/ChannelManager.js b/services/real-time/app/js/ChannelManager.js index 1ca9f6e88b..c777fb2250 100644 --- a/services/real-time/app/js/ChannelManager.js +++ b/services/real-time/app/js/ChannelManager.js @@ -1,6 +1,6 @@ const logger = require('logger-sharelatex') const metrics = require('@overleaf/metrics') -const settings = require('settings-sharelatex') +const settings = require('@overleaf/settings') const OError = require('@overleaf/o-error') const ClientMap = new Map() // for each redis client, store a Map of subscribed channels (channelname -> subscribe promise) diff --git a/services/real-time/app/js/ConnectedUsersManager.js b/services/real-time/app/js/ConnectedUsersManager.js index 6b98043ff7..006e763645 100644 --- a/services/real-time/app/js/ConnectedUsersManager.js +++ b/services/real-time/app/js/ConnectedUsersManager.js @@ -2,7 +2,7 @@ camelcase, */ const async = require('async') -const Settings = require('settings-sharelatex') +const Settings = require('@overleaf/settings') const logger = require('logger-sharelatex') const redis = require('@overleaf/redis-wrapper') const OError = require('@overleaf/o-error') diff --git a/services/real-time/app/js/DeploymentManager.js b/services/real-time/app/js/DeploymentManager.js index ddb98fd4ce..fd482740f5 100644 --- a/services/real-time/app/js/DeploymentManager.js +++ b/services/real-time/app/js/DeploymentManager.js @@ -1,5 +1,5 @@ const logger = require('logger-sharelatex') -const settings = require('settings-sharelatex') +const settings = require('@overleaf/settings') const fs = require('fs') // Monitor a status file (e.g. /etc/real_time_status) periodically and close the diff --git a/services/real-time/app/js/DocumentUpdaterController.js b/services/real-time/app/js/DocumentUpdaterController.js index 0e51339600..7ce516524c 100644 --- a/services/real-time/app/js/DocumentUpdaterController.js +++ b/services/real-time/app/js/DocumentUpdaterController.js @@ -2,7 +2,7 @@ camelcase, */ const logger = require('logger-sharelatex') -const settings = require('settings-sharelatex') +const settings = require('@overleaf/settings') const RedisClientManager = require('./RedisClientManager') const SafeJsonParse = require('./SafeJsonParse') const EventLogger = require('./EventLogger') diff --git a/services/real-time/app/js/DocumentUpdaterManager.js b/services/real-time/app/js/DocumentUpdaterManager.js index 39bef96ebd..715d1a70d3 100644 --- a/services/real-time/app/js/DocumentUpdaterManager.js +++ b/services/real-time/app/js/DocumentUpdaterManager.js @@ -5,7 +5,7 @@ const request = require('request') const _ = require('underscore') const OError = require('@overleaf/o-error') const logger = require('logger-sharelatex') -const settings = require('settings-sharelatex') +const settings = require('@overleaf/settings') const metrics = require('@overleaf/metrics') const { ClientRequestedMissingOpsError, diff --git a/services/real-time/app/js/EventLogger.js b/services/real-time/app/js/EventLogger.js index 1a2d898577..e6baffaa50 100644 --- a/services/real-time/app/js/EventLogger.js +++ b/services/real-time/app/js/EventLogger.js @@ -4,7 +4,7 @@ let EventLogger const logger = require('logger-sharelatex') const metrics = require('@overleaf/metrics') -const settings = require('settings-sharelatex') +const settings = require('@overleaf/settings') // keep track of message counters to detect duplicate and out of order events // messsage ids have the format "UNIQUEHOSTKEY-COUNTER" diff --git a/services/real-time/app/js/Router.js b/services/real-time/app/js/Router.js index 28765ecfa8..85b48eaf66 100644 --- a/services/real-time/app/js/Router.js +++ b/services/real-time/app/js/Router.js @@ -3,7 +3,7 @@ */ const metrics = require('@overleaf/metrics') const logger = require('logger-sharelatex') -const settings = require('settings-sharelatex') +const settings = require('@overleaf/settings') const WebsocketController = require('./WebsocketController') const HttpController = require('./HttpController') const HttpApiController = require('./HttpApiController') diff --git a/services/real-time/app/js/SafeJsonParse.js b/services/real-time/app/js/SafeJsonParse.js index a8a3afae4d..b66f032ec6 100644 --- a/services/real-time/app/js/SafeJsonParse.js +++ b/services/real-time/app/js/SafeJsonParse.js @@ -1,4 +1,4 @@ -const Settings = require('settings-sharelatex') +const Settings = require('@overleaf/settings') const { DataTooLargeToParseError } = require('./Errors') module.exports = { diff --git a/services/real-time/app/js/WebApiManager.js b/services/real-time/app/js/WebApiManager.js index 403de53cfe..c5ac369149 100644 --- a/services/real-time/app/js/WebApiManager.js +++ b/services/real-time/app/js/WebApiManager.js @@ -3,7 +3,7 @@ */ const request = require('request') const OError = require('@overleaf/o-error') -const settings = require('settings-sharelatex') +const settings = require('@overleaf/settings') const logger = require('logger-sharelatex') const { CodedError, diff --git a/services/real-time/app/js/WebsocketLoadBalancer.js b/services/real-time/app/js/WebsocketLoadBalancer.js index ee6ba4d335..4f1a4935a7 100644 --- a/services/real-time/app/js/WebsocketLoadBalancer.js +++ b/services/real-time/app/js/WebsocketLoadBalancer.js @@ -1,7 +1,7 @@ /* eslint-disable camelcase, */ -const Settings = require('settings-sharelatex') +const Settings = require('@overleaf/settings') const logger = require('logger-sharelatex') const RedisClientManager = require('./RedisClientManager') const SafeJsonParse = require('./SafeJsonParse') diff --git a/services/real-time/package-lock.json b/services/real-time/package-lock.json index ba777f5d4c..85a0167a95 100644 --- a/services/real-time/package-lock.json +++ b/services/real-time/package-lock.json @@ -703,6 +703,11 @@ "ioredis": "~4.17.3" } }, + "@overleaf/settings": { + "version": "2.1.1", + "resolved": "https://registry.npmjs.org/@overleaf/settings/-/settings-2.1.1.tgz", + "integrity": "sha512-vcJwqCGFKmQxTP/syUqCeMaSRjHmBcQgKOACR9He2uJcErg2GZPa1go+nGvszMbkElM4HfRKm/MfxvqHhoN4TQ==" + }, "@protobufjs/aspromise": { "version": "1.1.2", "resolved": "https://registry.npmjs.org/@protobufjs/aspromise/-/aspromise-1.1.2.tgz", @@ -5918,21 +5923,6 @@ "resolved": "https://registry.npmjs.org/setprototypeof/-/setprototypeof-1.1.1.tgz", "integrity": "sha512-JvdAWfbXeIGaZ9cILp38HntZSFSo3mWg6xGcJJsd+d4aRMOqauag1C63dJfDw7OaMYwEbHMOxEZ1lqVRYP2OAw==" }, - "settings-sharelatex": { - "version": "1.1.0", - "resolved": "https://registry.npmjs.org/settings-sharelatex/-/settings-sharelatex-1.1.0.tgz", - "integrity": "sha512-f7D+0lnlohoteSn6IKTH72NE+JnAdMWTKwQglAuimZWTID2FRRItZSGeYMTRpvEnaQApkoVwRp//WRMsiddnqw==", - "requires": { - "coffee-script": "1.6.0" - }, - "dependencies": { - "coffee-script": { - "version": "1.6.0", - "resolved": "https://registry.npmjs.org/coffee-script/-/coffee-script-1.6.0.tgz", - "integrity": "sha512-Tx8itEfCsQp8RbLDFt7qwjqXycAx2g6SI7//4PPUR2j6meLmNifYm6zKrNDcU1+Q/GWRhjhEZk7DaLG1TfIzGA==" - } - } - }, "shebang-command": { "version": "1.2.0", "resolved": "https://registry.npmjs.org/shebang-command/-/shebang-command-1.2.0.tgz", diff --git a/services/real-time/package.json b/services/real-time/package.json index f07ab3da62..37cd18f81f 100644 --- a/services/real-time/package.json +++ b/services/real-time/package.json @@ -24,6 +24,7 @@ "@overleaf/metrics": "^3.5.1", "@overleaf/o-error": "^3.1.0", "@overleaf/redis-wrapper": "^2.0.0", + "@overleaf/settings": "^2.1.1", "async": "^0.9.0", "base64id": "0.1.0", "basic-auth-connect": "^1.0.0", @@ -35,7 +36,6 @@ "express-session": "^1.17.1", "logger-sharelatex": "^2.2.0", "request": "^2.88.2", - "settings-sharelatex": "^1.1.0", "socket.io": "https://github.com/overleaf/socket.io/archive/0.9.19-overleaf-5.tar.gz", "socket.io-client": "https://github.com/overleaf/socket.io-client/archive/0.9.17-overleaf-3.tar.gz", "underscore": "1.13.1" diff --git a/services/real-time/test/acceptance/js/ApplyUpdateTests.js b/services/real-time/test/acceptance/js/ApplyUpdateTests.js index 14a8faa186..785afa3bf4 100644 --- a/services/real-time/test/acceptance/js/ApplyUpdateTests.js +++ b/services/real-time/test/acceptance/js/ApplyUpdateTests.js @@ -18,7 +18,7 @@ const { expect } = require('chai') const RealTimeClient = require('./helpers/RealTimeClient') const FixturesManager = require('./helpers/FixturesManager') -const settings = require('settings-sharelatex') +const settings = require('@overleaf/settings') const redis = require('@overleaf/redis-wrapper') const rclient = redis.createClient(settings.redis.documentupdater) diff --git a/services/real-time/test/acceptance/js/DrainManagerTests.js b/services/real-time/test/acceptance/js/DrainManagerTests.js index d312d34aa9..f4d5636adb 100644 --- a/services/real-time/test/acceptance/js/DrainManagerTests.js +++ b/services/real-time/test/acceptance/js/DrainManagerTests.js @@ -16,7 +16,7 @@ const { expect } = require('chai') const async = require('async') const request = require('request') -const Settings = require('settings-sharelatex') +const Settings = require('@overleaf/settings') const drain = function (rate, callback) { request.post( diff --git a/services/real-time/test/acceptance/js/EarlyDisconnect.js b/services/real-time/test/acceptance/js/EarlyDisconnect.js index b6d360d0ea..cce2b318ec 100644 --- a/services/real-time/test/acceptance/js/EarlyDisconnect.js +++ b/services/real-time/test/acceptance/js/EarlyDisconnect.js @@ -18,7 +18,7 @@ const MockDocUpdaterServer = require('./helpers/MockDocUpdaterServer') const MockWebServer = require('./helpers/MockWebServer') const FixturesManager = require('./helpers/FixturesManager') -const settings = require('settings-sharelatex') +const settings = require('@overleaf/settings') const redis = require('@overleaf/redis-wrapper') const rclient = redis.createClient(settings.redis.pubsub) const rclientRT = redis.createClient(settings.redis.realtime) diff --git a/services/real-time/test/acceptance/js/LeaveProjectTests.js b/services/real-time/test/acceptance/js/LeaveProjectTests.js index 8364e25c4a..0d30c90c8f 100644 --- a/services/real-time/test/acceptance/js/LeaveProjectTests.js +++ b/services/real-time/test/acceptance/js/LeaveProjectTests.js @@ -17,7 +17,7 @@ const FixturesManager = require('./helpers/FixturesManager') const async = require('async') -const settings = require('settings-sharelatex') +const settings = require('@overleaf/settings') const redis = require('@overleaf/redis-wrapper') const rclient = redis.createClient(settings.redis.pubsub) diff --git a/services/real-time/test/acceptance/js/MatrixTests.js b/services/real-time/test/acceptance/js/MatrixTests.js index 9da735083e..ee7686679a 100644 --- a/services/real-time/test/acceptance/js/MatrixTests.js +++ b/services/real-time/test/acceptance/js/MatrixTests.js @@ -53,7 +53,7 @@ const async = require('async') const RealTimeClient = require('./helpers/RealTimeClient') const FixturesManager = require('./helpers/FixturesManager') -const settings = require('settings-sharelatex') +const settings = require('@overleaf/settings') const Keys = settings.redis.documentupdater.key_schema const redis = require('@overleaf/redis-wrapper') const rclient = redis.createClient(settings.redis.pubsub) diff --git a/services/real-time/test/acceptance/js/PubSubRace.js b/services/real-time/test/acceptance/js/PubSubRace.js index a331f73c1b..db97a3d00e 100644 --- a/services/real-time/test/acceptance/js/PubSubRace.js +++ b/services/real-time/test/acceptance/js/PubSubRace.js @@ -15,7 +15,7 @@ const FixturesManager = require('./helpers/FixturesManager') const async = require('async') -const settings = require('settings-sharelatex') +const settings = require('@overleaf/settings') const redis = require('@overleaf/redis-wrapper') const rclient = redis.createClient(settings.redis.pubsub) diff --git a/services/real-time/test/acceptance/js/ReceiveUpdateTests.js b/services/real-time/test/acceptance/js/ReceiveUpdateTests.js index ae7ab8ef55..92d29b014f 100644 --- a/services/real-time/test/acceptance/js/ReceiveUpdateTests.js +++ b/services/real-time/test/acceptance/js/ReceiveUpdateTests.js @@ -19,7 +19,7 @@ const FixturesManager = require('./helpers/FixturesManager') const async = require('async') -const settings = require('settings-sharelatex') +const settings = require('@overleaf/settings') const redis = require('@overleaf/redis-wrapper') const rclient = redis.createClient(settings.redis.pubsub) diff --git a/services/real-time/test/acceptance/js/SessionSocketsTests.js b/services/real-time/test/acceptance/js/SessionSocketsTests.js index 45f62195e5..44835142ea 100644 --- a/services/real-time/test/acceptance/js/SessionSocketsTests.js +++ b/services/real-time/test/acceptance/js/SessionSocketsTests.js @@ -9,7 +9,7 @@ * Full docs: https://github.com/decaffeinate/decaffeinate/blob/master/docs/suggestions.md */ const RealTimeClient = require('./helpers/RealTimeClient') -const Settings = require('settings-sharelatex') +const Settings = require('@overleaf/settings') const { expect } = require('chai') describe('SessionSockets', function () { diff --git a/services/real-time/test/acceptance/js/helpers/RealTimeClient.js b/services/real-time/test/acceptance/js/helpers/RealTimeClient.js index 03589e4fa0..d5e8b79343 100644 --- a/services/real-time/test/acceptance/js/helpers/RealTimeClient.js +++ b/services/real-time/test/acceptance/js/helpers/RealTimeClient.js @@ -17,7 +17,7 @@ const io = require('socket.io-client') const async = require('async') const request = require('request') -const Settings = require('settings-sharelatex') +const Settings = require('@overleaf/settings') const redis = require('@overleaf/redis-wrapper') const rclient = redis.createClient(Settings.redis.websessions) diff --git a/services/real-time/test/acceptance/js/helpers/RealtimeServer.js b/services/real-time/test/acceptance/js/helpers/RealtimeServer.js index 950c4b966d..ef1a85a2a5 100644 --- a/services/real-time/test/acceptance/js/helpers/RealtimeServer.js +++ b/services/real-time/test/acceptance/js/helpers/RealtimeServer.js @@ -14,7 +14,7 @@ */ const app = require('../../../../app') const logger = require('logger-sharelatex') -const Settings = require('settings-sharelatex') +const Settings = require('@overleaf/settings') module.exports = { running: false, diff --git a/services/real-time/test/unit/js/ChannelManagerTests.js b/services/real-time/test/unit/js/ChannelManagerTests.js index aad5ddbd42..a7d857dada 100644 --- a/services/real-time/test/unit/js/ChannelManagerTests.js +++ b/services/real-time/test/unit/js/ChannelManagerTests.js @@ -20,7 +20,7 @@ describe('ChannelManager', function () { this.other_rclient = {} return (this.ChannelManager = SandboxedModule.require(modulePath, { requires: { - 'settings-sharelatex': (this.settings = {}), + '@overleaf/settings': (this.settings = {}), '@overleaf/metrics': (this.metrics = { inc: sinon.stub(), summary: sinon.stub() diff --git a/services/real-time/test/unit/js/ConnectedUsersManagerTests.js b/services/real-time/test/unit/js/ConnectedUsersManagerTests.js index a0ab01631e..8525b94b02 100644 --- a/services/real-time/test/unit/js/ConnectedUsersManagerTests.js +++ b/services/real-time/test/unit/js/ConnectedUsersManagerTests.js @@ -56,7 +56,7 @@ describe('ConnectedUsersManager', function () { this.ConnectedUsersManager = SandboxedModule.require(modulePath, { requires: { - 'settings-sharelatex': this.settings, + '@overleaf/settings': this.settings, '@overleaf/redis-wrapper': { createClient: () => { return this.rClient diff --git a/services/real-time/test/unit/js/DocumentUpdaterControllerTests.js b/services/real-time/test/unit/js/DocumentUpdaterControllerTests.js index 8f0be7f87d..333fe53499 100644 --- a/services/real-time/test/unit/js/DocumentUpdaterControllerTests.js +++ b/services/real-time/test/unit/js/DocumentUpdaterControllerTests.js @@ -28,7 +28,7 @@ describe('DocumentUpdaterController', function () { this.RoomEvents = { on: sinon.stub() } this.EditorUpdatesController = SandboxedModule.require(modulePath, { requires: { - 'settings-sharelatex': (this.settings = { + '@overleaf/settings': (this.settings = { redis: { documentupdater: { key_schema: { diff --git a/services/real-time/test/unit/js/DocumentUpdaterManagerTests.js b/services/real-time/test/unit/js/DocumentUpdaterManagerTests.js index 94ef11f716..e2b6bdbe51 100644 --- a/services/real-time/test/unit/js/DocumentUpdaterManagerTests.js +++ b/services/real-time/test/unit/js/DocumentUpdaterManagerTests.js @@ -41,7 +41,7 @@ describe('DocumentUpdaterManager', function () { return (this.DocumentUpdaterManager = SandboxedModule.require(modulePath, { requires: { - 'settings-sharelatex': this.settings, + '@overleaf/settings': this.settings, request: (this.request = {}), '@overleaf/redis-wrapper': { createClient: () => this.rclient }, '@overleaf/metrics': (this.Metrics = { diff --git a/services/real-time/test/unit/js/RoomManagerTests.js b/services/real-time/test/unit/js/RoomManagerTests.js index fd22271469..b2cd2fe0e3 100644 --- a/services/real-time/test/unit/js/RoomManagerTests.js +++ b/services/real-time/test/unit/js/RoomManagerTests.js @@ -23,7 +23,7 @@ describe('RoomManager', function () { this.client = { namespace: { name: '' }, id: 'first-client' } this.RoomManager = SandboxedModule.require(modulePath, { requires: { - 'settings-sharelatex': (this.settings = {}), + '@overleaf/settings': (this.settings = {}), '@overleaf/metrics': (this.metrics = { gauge: sinon.stub() }) } }) diff --git a/services/real-time/test/unit/js/SafeJsonParseTest.js b/services/real-time/test/unit/js/SafeJsonParseTest.js index de50bccebb..bdbba00c93 100644 --- a/services/real-time/test/unit/js/SafeJsonParseTest.js +++ b/services/real-time/test/unit/js/SafeJsonParseTest.js @@ -19,7 +19,7 @@ describe('SafeJsonParse', function () { beforeEach(function () { return (this.SafeJsonParse = SandboxedModule.require(modulePath, { requires: { - 'settings-sharelatex': (this.Settings = { + '@overleaf/settings': (this.Settings = { maxUpdateSize: 16 * 1024 }) } diff --git a/services/real-time/test/unit/js/WebApiManagerTests.js b/services/real-time/test/unit/js/WebApiManagerTests.js index 97be3c27f0..b1291b4205 100644 --- a/services/real-time/test/unit/js/WebApiManagerTests.js +++ b/services/real-time/test/unit/js/WebApiManagerTests.js @@ -23,7 +23,7 @@ describe('WebApiManager', function () { return (this.WebApiManager = SandboxedModule.require(modulePath, { requires: { request: (this.request = {}), - 'settings-sharelatex': (this.settings = { + '@overleaf/settings': (this.settings = { apis: { web: { url: 'http://web.example.com', From e66d1231f24a5528c35100bc5888f04e40c99a5d Mon Sep 17 00:00:00 2001 From: Jakob Ackermann Date: Mon, 12 Jul 2021 17:51:04 +0100 Subject: [PATCH 484/491] [misc] run npm dedupe --- services/real-time/package-lock.json | 213 +++------------------------ 1 file changed, 20 insertions(+), 193 deletions(-) diff --git a/services/real-time/package-lock.json b/services/real-time/package-lock.json index 85a0167a95..87bfbe62ff 100644 --- a/services/real-time/package-lock.json +++ b/services/real-time/package-lock.json @@ -175,17 +175,8 @@ "resolved": "https://registry.npmjs.org/@google-cloud/promisify/-/promisify-1.0.4.tgz", "integrity": "sha512-VccZDcOql77obTnFh0TbNED/6ZbbmHDf8UMNnzO1d5g9V0Htfm4k5cllY8P1tJsRKC3zWYGRLaViiupcgVjBoQ==" }, - "agent-base": { - "version": "6.0.1", - "resolved": "https://registry.npmjs.org/agent-base/-/agent-base-6.0.1.tgz", - "integrity": "sha512-01q25QQDwLSsyfhrKbn8yuur+JNw0H+0Y4JiGIKd3z9aYk/w/2kxD/Upc+t2ZBBSUNff50VjPsSW2YxM8QYKVg==", - "requires": { - "debug": "4" - } - }, "debug": { - "version": "4.1.1", - "resolved": "https://registry.npmjs.org/debug/-/debug-4.1.1.tgz", + "version": "https://registry.npmjs.org/debug/-/debug-4.1.1.tgz", "integrity": "sha512-pYAIzeRo8J6KPEaJ0VWOh5Pzkbw/RetuzehGM7QRRX5he4fPHx2rdKMB256ehJCkX+XRQm16eZLqLNS8RSZXZw==", "requires": { "ms": "^2.1.1" @@ -247,34 +238,6 @@ "mime": "^2.2.0" } }, - "https-proxy-agent": { - "version": "5.0.0", - "resolved": "https://registry.npmjs.org/https-proxy-agent/-/https-proxy-agent-5.0.0.tgz", - "integrity": "sha512-EkYm5BcKUGiduxzSt3Eppko+PiNWNEpa4ySk9vTC6wDsQJW9rHSa+UhGNJoRYp7bz6Ht1eaRIa6QaJqO5rCFbA==", - "requires": { - "agent-base": "6", - "debug": "4" - } - }, - "jwa": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/jwa/-/jwa-2.0.0.tgz", - "integrity": "sha512-jrZ2Qx916EA+fq9cEAeCROWPTfCwi1IVHqT2tapuqLEVVDKFDENFw1oL+MwrTvH6msKxsd1YTDVw6uKEcsrLEA==", - "requires": { - "buffer-equal-constant-time": "1.0.1", - "ecdsa-sig-formatter": "1.0.11", - "safe-buffer": "^5.0.1" - } - }, - "jws": { - "version": "4.0.0", - "resolved": "https://registry.npmjs.org/jws/-/jws-4.0.0.tgz", - "integrity": "sha512-KDncfTmOZoOMTFG4mBlG0qUIOlc03fmzH+ru6RgYVZhPkyiy/92Owlt/8UEN+a4TXR1FQetfIpJE8ApdvdVxTg==", - "requires": { - "jwa": "^2.0.0", - "safe-buffer": "^5.0.1" - } - }, "mime": { "version": "2.4.6", "resolved": "https://registry.npmjs.org/mime/-/mime-2.4.6.tgz", @@ -318,22 +281,13 @@ "google-auth-library": "^6.0.0" }, "dependencies": { - "agent-base": { - "version": "6.0.1", - "resolved": "https://registry.npmjs.org/agent-base/-/agent-base-6.0.1.tgz", - "integrity": "sha512-01q25QQDwLSsyfhrKbn8yuur+JNw0H+0Y4JiGIKd3z9aYk/w/2kxD/Upc+t2ZBBSUNff50VjPsSW2YxM8QYKVg==", - "requires": { - "debug": "4" - } - }, "bignumber.js": { "version": "9.0.0", "resolved": "https://registry.npmjs.org/bignumber.js/-/bignumber.js-9.0.0.tgz", "integrity": "sha512-t/OYhhJ2SD+YGBQcjY8GzzDHEk9f3nerxjtfa6tlMXfe7frs/WozhvCNoGvpM0P3bNf3Gq5ZRMlGr5f3r4/N8A==" }, "debug": { - "version": "4.1.1", - "resolved": "https://registry.npmjs.org/debug/-/debug-4.1.1.tgz", + "version": "https://registry.npmjs.org/debug/-/debug-4.1.1.tgz", "integrity": "sha512-pYAIzeRo8J6KPEaJ0VWOh5Pzkbw/RetuzehGM7QRRX5he4fPHx2rdKMB256ehJCkX+XRQm16eZLqLNS8RSZXZw==", "requires": { "ms": "^2.1.1" @@ -351,15 +305,6 @@ "node-fetch": "^2.3.0" } }, - "gcp-metadata": { - "version": "4.1.4", - "resolved": "https://registry.npmjs.org/gcp-metadata/-/gcp-metadata-4.1.4.tgz", - "integrity": "sha512-5J/GIH0yWt/56R3dNaNWPGQ/zXsZOddYECfJaqxFWgrZ9HC2Kvc5vl9upOgUUHKzURjAVf2N+f6tEJiojqXUuA==", - "requires": { - "gaxios": "^3.0.0", - "json-bigint": "^1.0.0" - } - }, "google-auth-library": { "version": "6.0.6", "resolved": "https://registry.npmjs.org/google-auth-library/-/google-auth-library-6.0.6.tgz", @@ -376,61 +321,13 @@ "lru-cache": "^6.0.0" } }, - "google-p12-pem": { - "version": "3.0.2", - "resolved": "https://registry.npmjs.org/google-p12-pem/-/google-p12-pem-3.0.2.tgz", - "integrity": "sha512-tbjzndQvSIHGBLzHnhDs3cL4RBjLbLXc2pYvGH+imGVu5b4RMAttUTdnmW2UH0t11QeBTXZ7wlXPS7hrypO/tg==", - "requires": { - "node-forge": "^0.9.0" - } - }, - "gtoken": { - "version": "5.0.3", - "resolved": "https://registry.npmjs.org/gtoken/-/gtoken-5.0.3.tgz", - "integrity": "sha512-Nyd1wZCMRc2dj/mAD0LlfQLcAO06uKdpKJXvK85SGrF5+5+Bpfil9u/2aw35ltvEHjvl0h5FMKN5knEU+9JrOg==", - "requires": { - "gaxios": "^3.0.0", - "google-p12-pem": "^3.0.0", - "jws": "^4.0.0", - "mime": "^2.2.0" - } - }, - "https-proxy-agent": { - "version": "5.0.0", - "resolved": "https://registry.npmjs.org/https-proxy-agent/-/https-proxy-agent-5.0.0.tgz", - "integrity": "sha512-EkYm5BcKUGiduxzSt3Eppko+PiNWNEpa4ySk9vTC6wDsQJW9rHSa+UhGNJoRYp7bz6Ht1eaRIa6QaJqO5rCFbA==", - "requires": { - "agent-base": "6", - "debug": "4" - } - }, "json-bigint": { - "version": "1.0.0", - "resolved": "https://registry.npmjs.org/json-bigint/-/json-bigint-1.0.0.tgz", + "version": "https://registry.npmjs.org/json-bigint/-/json-bigint-1.0.0.tgz", "integrity": "sha512-SiPv/8VpZuWbvLSMtTDU8hEfrZWg/mH/nV/b4o0CYbSxu1UIQPLdwKOCIyLQX+VIPO5vrLX3i8qtqFyhdPSUSQ==", "requires": { "bignumber.js": "^9.0.0" } }, - "jwa": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/jwa/-/jwa-2.0.0.tgz", - "integrity": "sha512-jrZ2Qx916EA+fq9cEAeCROWPTfCwi1IVHqT2tapuqLEVVDKFDENFw1oL+MwrTvH6msKxsd1YTDVw6uKEcsrLEA==", - "requires": { - "buffer-equal-constant-time": "1.0.1", - "ecdsa-sig-formatter": "1.0.11", - "safe-buffer": "^5.0.1" - } - }, - "jws": { - "version": "4.0.0", - "resolved": "https://registry.npmjs.org/jws/-/jws-4.0.0.tgz", - "integrity": "sha512-KDncfTmOZoOMTFG4mBlG0qUIOlc03fmzH+ru6RgYVZhPkyiy/92Owlt/8UEN+a4TXR1FQetfIpJE8ApdvdVxTg==", - "requires": { - "jwa": "^2.0.0", - "safe-buffer": "^5.0.1" - } - }, "lru-cache": { "version": "6.0.0", "resolved": "https://registry.npmjs.org/lru-cache/-/lru-cache-6.0.0.tgz", @@ -440,8 +337,7 @@ } }, "mime": { - "version": "2.4.6", - "resolved": "https://registry.npmjs.org/mime/-/mime-2.4.6.tgz", + "version": "https://registry.npmjs.org/mime/-/mime-2.4.6.tgz", "integrity": "sha512-RZKhC3EmpBchfTGBVb8fb+RL2cWyw/32lshnsETttkBAyAUXSGHxbEJWWRXc751DrIxG1q04b8QwMbAwkRPpUA==" }, "ms": { @@ -450,8 +346,7 @@ "integrity": "sha512-sGkPx+VjMtmA6MX27oA4FBFELFCZZ4S4XqeGOXCv68tT+jb3vk/RyaKWP0PTKyWtmLSM0b+adUTEvbs1PEaH2w==" }, "node-forge": { - "version": "0.9.1", - "resolved": "https://registry.npmjs.org/node-forge/-/node-forge-0.9.1.tgz", + "version": "https://registry.npmjs.org/node-forge/-/node-forge-0.9.1.tgz", "integrity": "sha512-G6RlQt5Sb4GMBzXvhfkeFmbqR6MzhtnT7VTHuLadjkii3rdYHNdw0m8zA4BTxVIh68FicCQ2NSUANpsqkr9jvQ==" }, "yallist": { @@ -1381,7 +1276,6 @@ "requires": { "anymatch": "~3.1.1", "braces": "~3.0.2", - "fsevents": "~2.3.1", "glob-parent": "~5.1.0", "is-binary-path": "~2.1.0", "is-glob": "~4.0.1", @@ -2573,13 +2467,6 @@ "resolved": "https://registry.npmjs.org/fs.realpath/-/fs.realpath-1.0.0.tgz", "integrity": "sha512-OO0pH2lK6a0hZnAdau5ItzHPI6pUlvI7jMVnxUQRtw4owF2wk8lOSabtGDCTP4Ggrg2MbGnWO9X8K1t4+fGMDw==" }, - "fsevents": { - "version": "2.3.2", - "resolved": "https://registry.npmjs.org/fsevents/-/fsevents-2.3.2.tgz", - "integrity": "sha512-xiqMQR4xAeHTuB9uWm+fFRcIOgKBMiOBP+eXiyT7jsgVCq1bkVygt00oASowB7EdtpOHaaPgKt812P9ab+DDKA==", - "dev": true, - "optional": true - }, "function-bind": { "version": "1.1.1", "resolved": "https://registry.npmjs.org/function-bind/-/function-bind-1.1.1.tgz", @@ -2780,17 +2667,8 @@ "walkdir": "^0.4.0" }, "dependencies": { - "agent-base": { - "version": "6.0.1", - "resolved": "https://registry.npmjs.org/agent-base/-/agent-base-6.0.1.tgz", - "integrity": "sha512-01q25QQDwLSsyfhrKbn8yuur+JNw0H+0Y4JiGIKd3z9aYk/w/2kxD/Upc+t2ZBBSUNff50VjPsSW2YxM8QYKVg==", - "requires": { - "debug": "4" - } - }, "debug": { - "version": "4.1.1", - "resolved": "https://registry.npmjs.org/debug/-/debug-4.1.1.tgz", + "version": "https://registry.npmjs.org/debug/-/debug-4.1.1.tgz", "integrity": "sha512-pYAIzeRo8J6KPEaJ0VWOh5Pzkbw/RetuzehGM7QRRX5he4fPHx2rdKMB256ehJCkX+XRQm16eZLqLNS8RSZXZw==", "requires": { "ms": "^2.1.1" @@ -2852,34 +2730,6 @@ "mime": "^2.2.0" } }, - "https-proxy-agent": { - "version": "5.0.0", - "resolved": "https://registry.npmjs.org/https-proxy-agent/-/https-proxy-agent-5.0.0.tgz", - "integrity": "sha512-EkYm5BcKUGiduxzSt3Eppko+PiNWNEpa4ySk9vTC6wDsQJW9rHSa+UhGNJoRYp7bz6Ht1eaRIa6QaJqO5rCFbA==", - "requires": { - "agent-base": "6", - "debug": "4" - } - }, - "jwa": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/jwa/-/jwa-2.0.0.tgz", - "integrity": "sha512-jrZ2Qx916EA+fq9cEAeCROWPTfCwi1IVHqT2tapuqLEVVDKFDENFw1oL+MwrTvH6msKxsd1YTDVw6uKEcsrLEA==", - "requires": { - "buffer-equal-constant-time": "1.0.1", - "ecdsa-sig-formatter": "1.0.11", - "safe-buffer": "^5.0.1" - } - }, - "jws": { - "version": "4.0.0", - "resolved": "https://registry.npmjs.org/jws/-/jws-4.0.0.tgz", - "integrity": "sha512-KDncfTmOZoOMTFG4mBlG0qUIOlc03fmzH+ru6RgYVZhPkyiy/92Owlt/8UEN+a4TXR1FQetfIpJE8ApdvdVxTg==", - "requires": { - "jwa": "^2.0.0", - "safe-buffer": "^5.0.1" - } - }, "mime": { "version": "2.4.6", "resolved": "https://registry.npmjs.org/mime/-/mime-2.4.6.tgz", @@ -3023,14 +2873,6 @@ "debug": "4" }, "dependencies": { - "agent-base": { - "version": "6.0.1", - "resolved": "https://registry.npmjs.org/agent-base/-/agent-base-6.0.1.tgz", - "integrity": "sha512-01q25QQDwLSsyfhrKbn8yuur+JNw0H+0Y4JiGIKd3z9aYk/w/2kxD/Upc+t2ZBBSUNff50VjPsSW2YxM8QYKVg==", - "requires": { - "debug": "4" - } - }, "debug": { "version": "4.1.1", "resolved": "https://registry.npmjs.org/debug/-/debug-4.1.1.tgz", @@ -3281,6 +3123,11 @@ "binary-extensions": "^2.0.0" } }, + "is-buffer": { + "version": "1.1.6", + "resolved": "https://registry.npmjs.org/is-buffer/-/is-buffer-1.1.6.tgz", + "integrity": "sha512-NcdALwpXkTm5Zvvbk7owOUSvVvBKDgKP5/ewfXEznmQFfs4ZRmanOeKBTjRVjka3QFoN6XJ+9F3USqfHqTaU5w==" + }, "is-callable": { "version": "1.2.0", "resolved": "https://registry.npmjs.org/is-callable/-/is-callable-1.2.0.tgz", @@ -3662,19 +3509,6 @@ "yn": "^4.0.0" }, "dependencies": { - "dtrace-provider": { - "version": "0.8.8", - "resolved": "https://registry.npmjs.org/dtrace-provider/-/dtrace-provider-0.8.8.tgz", - "integrity": "sha512-b7Z7cNtHPhH9EJhNNbbeqTcXB8LGFFZhq1PGgEvpeHlzd36bhbdTWoE/Ba/YguqpBSlAPKnARWhVlhunCMwfxg==", - "requires": { - "nan": "^2.14.0" - } - }, - "nan": { - "version": "2.14.1", - "resolved": "https://registry.npmjs.org/nan/-/nan-2.14.1.tgz", - "integrity": "sha512-isWHgVjnFjh2x2yuJ/tj3JbwoHu3UC2dX5G/88Cm24yB6YopVgxvBObDY7n5xW6ExmFhJpSEQqFPvq9zaXc8Jw==" - }, "yn": { "version": "4.0.0", "resolved": "https://registry.npmjs.org/yn/-/yn-4.0.0.tgz", @@ -3784,13 +3618,6 @@ "charenc": "0.0.2", "crypt": "0.0.2", "is-buffer": "~1.1.6" - }, - "dependencies": { - "is-buffer": { - "version": "1.1.6", - "resolved": "https://registry.npmjs.org/is-buffer/-/is-buffer-1.1.6.tgz", - "integrity": "sha512-NcdALwpXkTm5Zvvbk7owOUSvVvBKDgKP5/ewfXEznmQFfs4ZRmanOeKBTjRVjka3QFoN6XJ+9F3USqfHqTaU5w==" - } } }, "media-typer": { @@ -5685,15 +5512,6 @@ "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.2.1.tgz", "integrity": "sha512-rp3So07KcdmmKbGvgaNxQSJr7bGVSVk5S9Eq1F+ppbRo70+YeaDxkw5Dd8NPN+GD6bjnYm2VuPuCXmpuYvmCXQ==" }, - "tough-cookie": { - "version": "2.5.0", - "resolved": "https://registry.npmjs.org/tough-cookie/-/tough-cookie-2.5.0.tgz", - "integrity": "sha512-nlLsUzgm1kfLXSXfRZMc1KLAugd4hqJHDTvc2hDIwS3mZAfMEuMbc03SujMF+GEcpaX/qboeycw6iO8JwVv2+g==", - "requires": { - "psl": "^1.1.28", - "punycode": "^2.1.1" - } - }, "uuid": { "version": "3.4.0", "resolved": "https://registry.npmjs.org/uuid/-/uuid-3.4.0.tgz", @@ -6396,6 +6214,15 @@ "resolved": "https://registry.npmjs.org/toidentifier/-/toidentifier-1.0.0.tgz", "integrity": "sha512-yaOH/Pk/VEhBWWTlhI+qXxDFXlejDGcQipMlyxda9nthulaxLZUNcUqFxokp0vcYnvteJln5FNQDRrxj3YcbVw==" }, + "tough-cookie": { + "version": "2.5.0", + "resolved": "https://registry.npmjs.org/tough-cookie/-/tough-cookie-2.5.0.tgz", + "integrity": "sha512-nlLsUzgm1kfLXSXfRZMc1KLAugd4hqJHDTvc2hDIwS3mZAfMEuMbc03SujMF+GEcpaX/qboeycw6iO8JwVv2+g==", + "requires": { + "psl": "^1.1.28", + "punycode": "^2.1.1" + } + }, "tsconfig-paths": { "version": "3.9.0", "resolved": "https://registry.npmjs.org/tsconfig-paths/-/tsconfig-paths-3.9.0.tgz", From 91fe61b1672550a0c636171eef7708c5d69bbc93 Mon Sep 17 00:00:00 2001 From: Jakob Ackermann Date: Tue, 13 Jul 2021 10:07:02 +0100 Subject: [PATCH 485/491] [misc] goodbye coffee-script --- services/real-time/package-lock.json | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/services/real-time/package-lock.json b/services/real-time/package-lock.json index 87bfbe62ff..3da4ce62ac 100644 --- a/services/real-time/package-lock.json +++ b/services/real-time/package-lock.json @@ -1276,6 +1276,7 @@ "requires": { "anymatch": "~3.1.1", "braces": "~3.0.2", + "fsevents": "~2.3.1", "glob-parent": "~5.1.0", "is-binary-path": "~2.1.0", "is-glob": "~4.0.1", @@ -2467,6 +2468,13 @@ "resolved": "https://registry.npmjs.org/fs.realpath/-/fs.realpath-1.0.0.tgz", "integrity": "sha512-OO0pH2lK6a0hZnAdau5ItzHPI6pUlvI7jMVnxUQRtw4owF2wk8lOSabtGDCTP4Ggrg2MbGnWO9X8K1t4+fGMDw==" }, + "fsevents": { + "version": "2.3.2", + "resolved": "https://registry.npmjs.org/fsevents/-/fsevents-2.3.2.tgz", + "integrity": "sha512-xiqMQR4xAeHTuB9uWm+fFRcIOgKBMiOBP+eXiyT7jsgVCq1bkVygt00oASowB7EdtpOHaaPgKt812P9ab+DDKA==", + "dev": true, + "optional": true + }, "function-bind": { "version": "1.1.1", "resolved": "https://registry.npmjs.org/function-bind/-/function-bind-1.1.1.tgz", From d17cda1a7ba30e93cb6c2634df55b284f29c18ba Mon Sep 17 00:00:00 2001 From: Jakob Ackermann Date: Tue, 13 Jul 2021 11:55:16 +0100 Subject: [PATCH 486/491] [misc] upgrade build scripts to version 3.11.0 and cleanup packages ``` npm uninstall prettier-eslint-cli eslint-plugin-standard eslint-plugin-jsx-a11y eslint-plugin-react eslint-config-standard-jsx eslint-config-standard-react babel-eslint npm dedupe ``` --- services/real-time/.eslintrc | 2 +- services/real-time/.github/dependabot.yml | 2 +- services/real-time/.prettierrc | 6 +- services/real-time/buildscript.txt | 2 +- services/real-time/package-lock.json | 2732 ++++++--------------- services/real-time/package.json | 30 +- 6 files changed, 804 insertions(+), 1970 deletions(-) diff --git a/services/real-time/.eslintrc b/services/real-time/.eslintrc index 321353f971..1c14f50efe 100644 --- a/services/real-time/.eslintrc +++ b/services/real-time/.eslintrc @@ -3,9 +3,9 @@ // https://github.com/sharelatex/sharelatex-dev-environment { "extends": [ + "eslint:recommended", "standard", "prettier", - "prettier/standard" ], "parserOptions": { "ecmaVersion": 2018 diff --git a/services/real-time/.github/dependabot.yml b/services/real-time/.github/dependabot.yml index e2c64a3351..c856753655 100644 --- a/services/real-time/.github/dependabot.yml +++ b/services/real-time/.github/dependabot.yml @@ -20,4 +20,4 @@ updates: # future if we reorganise teams labels: - "dependencies" - - "Team-Magma" + - "type:maintenance" diff --git a/services/real-time/.prettierrc b/services/real-time/.prettierrc index 24f9ec526f..c92c3526e7 100644 --- a/services/real-time/.prettierrc +++ b/services/real-time/.prettierrc @@ -2,6 +2,10 @@ # Instead run bin/update_build_scripts from # https://github.com/sharelatex/sharelatex-dev-environment { + "arrowParens": "avoid", "semi": false, - "singleQuote": true + "singleQuote": true, + "trailingComma": "es5", + "tabWidth": 2, + "useTabs": false } diff --git a/services/real-time/buildscript.txt b/services/real-time/buildscript.txt index d8fc7d8c7d..e536d601f3 100644 --- a/services/real-time/buildscript.txt +++ b/services/real-time/buildscript.txt @@ -5,4 +5,4 @@ real-time --env-pass-through= --node-version=12.21.0 --public-repo=True ---script-version=3.8.0 +--script-version=3.11.0 diff --git a/services/real-time/package-lock.json b/services/real-time/package-lock.json index 3da4ce62ac..34cea2a4ea 100644 --- a/services/real-time/package-lock.json +++ b/services/real-time/package-lock.json @@ -5,31 +5,77 @@ "requires": true, "dependencies": { "@babel/code-frame": { - "version": "7.10.3", - "resolved": "https://registry.npmjs.org/@babel/code-frame/-/code-frame-7.10.3.tgz", - "integrity": "sha512-fDx9eNW0qz0WkUeqL6tXEXzVlPh6Y5aCDEZesl0xBGA8ndRukX91Uk44ZqnkECp01NAZUdCAl+aiQNGi0k88Eg==", + "version": "7.12.11", + "resolved": "https://registry.npmjs.org/@babel/code-frame/-/code-frame-7.12.11.tgz", + "integrity": "sha512-Zt1yodBx1UcyiePMSkWnU4hPqhwq7hGi2nFL1LeA3EUl+q2LQx16MISgJ0+z7dnmgvP9QtIleuETGOiOH1RcIw==", "dev": true, "requires": { - "@babel/highlight": "^7.10.3" + "@babel/highlight": "^7.10.4" } }, "@babel/helper-validator-identifier": { - "version": "7.10.3", - "resolved": "https://registry.npmjs.org/@babel/helper-validator-identifier/-/helper-validator-identifier-7.10.3.tgz", - "integrity": "sha512-bU8JvtlYpJSBPuj1VUmKpFGaDZuLxASky3LhaKj3bmpSTY6VWooSM8msk+Z0CZoErFye2tlABF6yDkT3FOPAXw==", + "version": "7.14.5", + "resolved": "https://registry.npmjs.org/@babel/helper-validator-identifier/-/helper-validator-identifier-7.14.5.tgz", + "integrity": "sha512-5lsetuxCLilmVGyiLEfoHBRX8UCFD+1m2x3Rj97WrW3V7H3u4RWRXA4evMjImCsin2J2YT0QaVDGf+z8ondbAg==", "dev": true }, "@babel/highlight": { - "version": "7.10.3", - "resolved": "https://registry.npmjs.org/@babel/highlight/-/highlight-7.10.3.tgz", - "integrity": "sha512-Ih9B/u7AtgEnySE2L2F0Xm0GaM729XqqLfHkalTsbjXGyqmf/6M0Cu0WpvqueUlW+xk88BHw9Nkpj49naU+vWw==", + "version": "7.14.5", + "resolved": "https://registry.npmjs.org/@babel/highlight/-/highlight-7.14.5.tgz", + "integrity": "sha512-qf9u2WFWVV0MppaL877j2dBtQIDgmidgjGk5VIMw3OadXvYaXn66U1BFlH2t4+t3i+8PhedppRv+i40ABzd+gg==", "dev": true, "requires": { - "@babel/helper-validator-identifier": "^7.10.3", + "@babel/helper-validator-identifier": "^7.14.5", "chalk": "^2.0.0", "js-tokens": "^4.0.0" } }, + "@eslint/eslintrc": { + "version": "0.4.2", + "resolved": "https://registry.npmjs.org/@eslint/eslintrc/-/eslintrc-0.4.2.tgz", + "integrity": "sha512-8nmGq/4ycLpIwzvhI4tNDmQztZ8sp+hI7cyG8i1nQDhkAbRzHpXPidRAHlNvCZQpJTKw5ItIpMw9RSToGF00mg==", + "dev": true, + "requires": { + "ajv": "^6.12.4", + "debug": "^4.1.1", + "espree": "^7.3.0", + "globals": "^13.9.0", + "ignore": "^4.0.6", + "import-fresh": "^3.2.1", + "js-yaml": "^3.13.1", + "minimatch": "^3.0.4", + "strip-json-comments": "^3.1.1" + }, + "dependencies": { + "ajv": { + "version": "6.12.6", + "resolved": "https://registry.npmjs.org/ajv/-/ajv-6.12.6.tgz", + "integrity": "sha512-j3fVLgvTo527anyYyJOGTYJbG+vnnQYvE0m5mmkc1TK+nxAppkCLMIL0aZ4dblVCNoGShhm+kzE4ZUykBoMg4g==", + "dev": true, + "requires": { + "fast-deep-equal": "^3.1.1", + "fast-json-stable-stringify": "^2.0.0", + "json-schema-traverse": "^0.4.1", + "uri-js": "^4.2.2" + } + }, + "debug": { + "version": "4.3.2", + "resolved": "https://registry.npmjs.org/debug/-/debug-4.3.2.tgz", + "integrity": "sha512-mOp8wKcvj7XxC78zLgw/ZA+6TSgkoE2C/ienthhRD298T7UNwAg9diBpLRxC0mOezLl4B0xV7M0cCO6P/O0Xhw==", + "dev": true, + "requires": { + "ms": "2.1.2" + } + }, + "ms": { + "version": "2.1.2", + "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.2.tgz", + "integrity": "sha512-sGkPx+VjMtmA6MX27oA4FBFELFCZZ4S4XqeGOXCv68tT+jb3vk/RyaKWP0PTKyWtmLSM0b+adUTEvbs1PEaH2w==", + "dev": true + } + } + }, "@google-cloud/common": { "version": "3.6.0", "resolved": "https://registry.npmjs.org/@google-cloud/common/-/common-3.6.0.tgz", @@ -87,11 +133,6 @@ "split": "^1.0.0" }, "dependencies": { - "acorn": { - "version": "8.1.0", - "resolved": "https://registry.npmjs.org/acorn/-/acorn-8.1.0.tgz", - "integrity": "sha512-LWCF/Wn0nfHOmJ9rzQApGnxnvgfROzGilS8936rqN/lfcYkY9MYZzdMqN+2NJ4SlTc+m5HiSa+kNfDtI64dwUA==" - }, "lru-cache": { "version": "6.0.0", "resolved": "https://registry.npmjs.org/lru-cache/-/lru-cache-6.0.0.tgz", @@ -100,14 +141,6 @@ "yallist": "^4.0.0" } }, - "p-limit": { - "version": "3.1.0", - "resolved": "https://registry.npmjs.org/p-limit/-/p-limit-3.1.0.tgz", - "integrity": "sha512-TYOanM3wGwNGsZN2cVTYPArw454xnXj5qmWF1bEoAc4+cU/ol7GVh7odevjp1FNHduHc3KZMcFduxU5Xc6uJRQ==", - "requires": { - "yocto-queue": "^0.1.0" - } - }, "semver": { "version": "7.3.5", "resolved": "https://registry.npmjs.org/semver/-/semver-7.3.5.tgz", @@ -528,6 +561,40 @@ "protobufjs": "^6.8.6" } }, + "@humanwhocodes/config-array": { + "version": "0.5.0", + "resolved": "https://registry.npmjs.org/@humanwhocodes/config-array/-/config-array-0.5.0.tgz", + "integrity": "sha512-FagtKFz74XrTl7y6HCzQpwDfXP0yhxe9lHLD1UZxjvZIcbyRz8zTFF/yYNfSfzU414eDwZ1SrO0Qvtyf+wFMQg==", + "dev": true, + "requires": { + "@humanwhocodes/object-schema": "^1.2.0", + "debug": "^4.1.1", + "minimatch": "^3.0.4" + }, + "dependencies": { + "debug": { + "version": "4.3.2", + "resolved": "https://registry.npmjs.org/debug/-/debug-4.3.2.tgz", + "integrity": "sha512-mOp8wKcvj7XxC78zLgw/ZA+6TSgkoE2C/ienthhRD298T7UNwAg9diBpLRxC0mOezLl4B0xV7M0cCO6P/O0Xhw==", + "dev": true, + "requires": { + "ms": "2.1.2" + } + }, + "ms": { + "version": "2.1.2", + "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.2.tgz", + "integrity": "sha512-sGkPx+VjMtmA6MX27oA4FBFELFCZZ4S4XqeGOXCv68tT+jb3vk/RyaKWP0PTKyWtmLSM0b+adUTEvbs1PEaH2w==", + "dev": true + } + } + }, + "@humanwhocodes/object-schema": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/@humanwhocodes/object-schema/-/object-schema-1.2.0.tgz", + "integrity": "sha512-wdppn25U8z/2yiaT6YGquE6X8sSv7hNMWSXYSSU1jGv/yd6XqjXgTDJ8KP4NgjTXfJ3GbRjeeb8RTV7a/VpM+w==", + "dev": true + }, "@opencensus/core": { "version": "0.0.20", "resolved": "https://registry.npmjs.org/@opencensus/core/-/core-0.0.20.tgz", @@ -664,14 +731,6 @@ "dev": true, "requires": { "type-detect": "4.0.8" - }, - "dependencies": { - "type-detect": { - "version": "4.0.8", - "resolved": "https://registry.npmjs.org/type-detect/-/type-detect-4.0.8.tgz", - "integrity": "sha512-0fr/mIH1dlO+x7TlcMy+bIDqKPsw/70tVyeHW787goQjhmqaZe10uwLujubK9q9Lg6Fiho1KUKDYz0Z7k7g5/g==", - "dev": true - } } }, "@sinonjs/fake-timers": { @@ -692,14 +751,6 @@ "@sinonjs/commons": "^1.6.0", "lodash.get": "^4.4.2", "type-detect": "^4.0.8" - }, - "dependencies": { - "type-detect": { - "version": "4.0.8", - "resolved": "https://registry.npmjs.org/type-detect/-/type-detect-4.0.8.tgz", - "integrity": "sha512-0fr/mIH1dlO+x7TlcMy+bIDqKPsw/70tVyeHW787goQjhmqaZe10uwLujubK9q9Lg6Fiho1KUKDYz0Z7k7g5/g==", - "dev": true - } } }, "@sinonjs/text-encoding": { @@ -713,23 +764,11 @@ "resolved": "https://registry.npmjs.org/@tootallnate/once/-/once-1.1.2.tgz", "integrity": "sha512-RbzJvlNzmRq5c3O09UipeuXno4tA1FE6ikOjxZK0tuxVv3412l64l5t1W5pj4+rJq9vpkm/kwiR07aZXnsKPxw==" }, - "@types/color-name": { - "version": "1.1.1", - "resolved": "https://registry.npmjs.org/@types/color-name/-/color-name-1.1.1.tgz", - "integrity": "sha512-rr+OQyAjxze7GgWrSaJwydHStIhHq2lvY3BOC2Mj7KnzI7XK0Uw1TOOdI9lDoajEbSWLiYgoo4f1R51erQfhPQ==", - "dev": true - }, "@types/console-log-level": { "version": "1.4.0", "resolved": "https://registry.npmjs.org/@types/console-log-level/-/console-log-level-1.4.0.tgz", "integrity": "sha512-x+OscEQwcx5Biair4enH7ov9W+clcqUWaZRaxn5IkT4yNWWjRr2oiYDkY/x1uXSTVZOQ2xlbFQySaQGB+VdXGQ==" }, - "@types/eslint-visitor-keys": { - "version": "1.0.0", - "resolved": "https://registry.npmjs.org/@types/eslint-visitor-keys/-/eslint-visitor-keys-1.0.0.tgz", - "integrity": "sha512-OCutwjDZ4aFS6PB1UZ988C4YgwlBHJd6wCeQqaLdmadZ/7e+w79+hbMUFC1QXDNCmdyoRfAFdm0RypzwR+Qpag==", - "dev": true - }, "@types/fs-extra": { "version": "8.1.1", "resolved": "https://registry.npmjs.org/@types/fs-extra/-/fs-extra-8.1.1.tgz", @@ -738,18 +777,6 @@ "@types/node": "*" } }, - "@types/json-schema": { - "version": "7.0.5", - "resolved": "https://registry.npmjs.org/@types/json-schema/-/json-schema-7.0.5.tgz", - "integrity": "sha512-7+2BITlgjgDhH0vvwZU/HZJVyk+2XUlvxXe8dFMedNX/aMkaOq++rMAFXc0tM7ij15QaWlbdQASBR9dihi+bDQ==", - "dev": true - }, - "@types/json5": { - "version": "0.0.29", - "resolved": "https://registry.npmjs.org/@types/json5/-/json5-0.0.29.tgz", - "integrity": "sha512-dRLjCWHYg4oaA77cxO64oO+7JwCwnIzkZPdrrC71jQmQtlhM556pwKo5bUzqvZndkVbeFLIIi+9TC40JNF5hNQ==", - "dev": true - }, "@types/long": { "version": "4.0.1", "resolved": "https://registry.npmjs.org/@types/long/-/long-4.0.1.tgz", @@ -765,59 +792,6 @@ "resolved": "https://registry.npmjs.org/@types/semver/-/semver-7.3.4.tgz", "integrity": "sha512-+nVsLKlcUCeMzD2ufHEYuJ9a2ovstb6Dp52A5VsoKxDXgvE051XgHI/33I1EymwkRGQkwnA0LkhnUzituGs4EQ==" }, - "@typescript-eslint/experimental-utils": { - "version": "1.13.0", - "resolved": "https://registry.npmjs.org/@typescript-eslint/experimental-utils/-/experimental-utils-1.13.0.tgz", - "integrity": "sha512-zmpS6SyqG4ZF64ffaJ6uah6tWWWgZ8m+c54XXgwFtUv0jNz8aJAVx8chMCvnk7yl6xwn8d+d96+tWp7fXzTuDg==", - "dev": true, - "requires": { - "@types/json-schema": "^7.0.3", - "@typescript-eslint/typescript-estree": "1.13.0", - "eslint-scope": "^4.0.0" - }, - "dependencies": { - "eslint-scope": { - "version": "4.0.3", - "resolved": "https://registry.npmjs.org/eslint-scope/-/eslint-scope-4.0.3.tgz", - "integrity": "sha512-p7VutNr1O/QrxysMo3E45FjYDTeXBy0iTltPFNSqKAIfjDSXC+4dj+qfyuD8bfAXrW/y6lW3O76VaYNPKfpKrg==", - "dev": true, - "requires": { - "esrecurse": "^4.1.0", - "estraverse": "^4.1.1" - } - } - } - }, - "@typescript-eslint/parser": { - "version": "1.13.0", - "resolved": "https://registry.npmjs.org/@typescript-eslint/parser/-/parser-1.13.0.tgz", - "integrity": "sha512-ITMBs52PCPgLb2nGPoeT4iU3HdQZHcPaZVw+7CsFagRJHUhyeTgorEwHXhFf3e7Evzi8oujKNpHc8TONth8AdQ==", - "dev": true, - "requires": { - "@types/eslint-visitor-keys": "^1.0.0", - "@typescript-eslint/experimental-utils": "1.13.0", - "@typescript-eslint/typescript-estree": "1.13.0", - "eslint-visitor-keys": "^1.0.0" - } - }, - "@typescript-eslint/typescript-estree": { - "version": "1.13.0", - "resolved": "https://registry.npmjs.org/@typescript-eslint/typescript-estree/-/typescript-estree-1.13.0.tgz", - "integrity": "sha512-b5rCmd2e6DCC6tCTN9GSUAuxdYwCM/k/2wdjHGrIRGPSJotWMCe/dGpi66u42bhuh8q3QBzqM4TMA1GUUCJvdw==", - "dev": true, - "requires": { - "lodash.unescape": "4.0.1", - "semver": "5.5.0" - }, - "dependencies": { - "semver": { - "version": "5.5.0", - "resolved": "https://registry.npmjs.org/semver/-/semver-5.5.0.tgz", - "integrity": "sha512-4SJ3dm0WAwWy/NVeioZh5AntkdJoWKxHxcmyP622fOkgHa4z3R0TdBJICINyaSDE6uNwVc8gZr+ZinwZAH4xIA==", - "dev": true - } - } - }, "@ungap/promise-all-settled": { "version": "1.1.2", "resolved": "https://registry.npmjs.org/@ungap/promise-all-settled/-/promise-all-settled-1.1.2.tgz", @@ -862,15 +836,14 @@ } }, "acorn": { - "version": "6.4.1", - "resolved": "https://registry.npmjs.org/acorn/-/acorn-6.4.1.tgz", - "integrity": "sha512-ZVA9k326Nwrj3Cj9jlh3wGFutC2ZornPNARZwsNYqQYgN0EsV2d53w5RN/co65Ohn4sUAUtb1rSUAOD6XN9idA==", - "dev": true + "version": "8.1.0", + "resolved": "https://registry.npmjs.org/acorn/-/acorn-8.1.0.tgz", + "integrity": "sha512-LWCF/Wn0nfHOmJ9rzQApGnxnvgfROzGilS8936rqN/lfcYkY9MYZzdMqN+2NJ4SlTc+m5HiSa+kNfDtI64dwUA==" }, "acorn-jsx": { - "version": "5.2.0", - "resolved": "https://registry.npmjs.org/acorn-jsx/-/acorn-jsx-5.2.0.tgz", - "integrity": "sha512-HiUX/+K2YpkpJ+SzBffkM/AQ2YE03S0U1kjTLVpoJdhZMOWy8qvXVN9JdLqv2QsaQ6MPYQIuNmwD8zOiYUofLQ==", + "version": "5.3.2", + "resolved": "https://registry.npmjs.org/acorn-jsx/-/acorn-jsx-5.3.2.tgz", + "integrity": "sha512-rq9s+JNhf0IChjtDXxllJ7g41oZk5SlXtp0LHwyA5cejwn7vKmKp4pPri6YEePv2PU65sAsegbXtIinmDFDXgQ==", "dev": true }, "active-x-obfuscator": { @@ -921,23 +894,6 @@ "integrity": "sha512-JoX0apGbHaUJBNl6yF+p6JAFYZ666/hhCGKN5t9QFjbJQKUU/g8MNbFDbvfrgKXvI1QpZplPOnwIo99lX/AAmA==", "dev": true }, - "ansi-escapes": { - "version": "4.3.1", - "resolved": "https://registry.npmjs.org/ansi-escapes/-/ansi-escapes-4.3.1.tgz", - "integrity": "sha512-JWF7ocqNrp8u9oqpgV+wH5ftbt+cfvv+PTjOvKLT3AdYly/LmORARfEVT1iyjwN+4MqE5UmVKoAdIBqeoCHgLA==", - "dev": true, - "requires": { - "type-fest": "^0.11.0" - }, - "dependencies": { - "type-fest": { - "version": "0.11.0", - "resolved": "https://registry.npmjs.org/type-fest/-/type-fest-0.11.0.tgz", - "integrity": "sha512-OdjXJxnCN1AvyLSzeKIgXTXxV+99ZuXl3Hpo9XpJAv9MBcHrrJOQ5kV7ypXOuQie+AmWG25hLbiKdwYTifzcfQ==", - "dev": true - } - } - }, "ansi-regex": { "version": "5.0.0", "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-5.0.0.tgz", @@ -992,24 +948,27 @@ "integrity": "sha512-PCVAQswWemu6UdxsDFFX/+gVeYqKAod3D3UVm91jHwynguOwAvYPhx8nNlM++NqRcK6CxxpUafjmhIdKiHibqg==" }, "array-includes": { - "version": "3.1.1", - "resolved": "https://registry.npmjs.org/array-includes/-/array-includes-3.1.1.tgz", - "integrity": "sha512-c2VXaCHl7zPsvpkFsw4nxvFie4fh1ur9bpcgsVkIjqn0H/Xwdg+7fv3n2r/isyS8EBj5b06M9kHyZuIr4El6WQ==", + "version": "3.1.3", + "resolved": "https://registry.npmjs.org/array-includes/-/array-includes-3.1.3.tgz", + "integrity": "sha512-gcem1KlBU7c9rB+Rq8/3PPKsK2kjqeEBa3bD5kkQo4nYlOHQCJqIJFqBXDEfwaRuYTT4E+FxA9xez7Gf/e3Q7A==", "dev": true, "requires": { + "call-bind": "^1.0.2", "define-properties": "^1.1.3", - "es-abstract": "^1.17.0", + "es-abstract": "^1.18.0-next.2", + "get-intrinsic": "^1.1.1", "is-string": "^1.0.5" } }, "array.prototype.flat": { - "version": "1.2.3", - "resolved": "https://registry.npmjs.org/array.prototype.flat/-/array.prototype.flat-1.2.3.tgz", - "integrity": "sha512-gBlRZV0VSmfPIeWfuuy56XZMvbVfbEUnOXUvt3F/eUUUSyzlgLxhEX4YAEpxNAogRGehPSnfXyPtYyKAhkzQhQ==", + "version": "1.2.4", + "resolved": "https://registry.npmjs.org/array.prototype.flat/-/array.prototype.flat-1.2.4.tgz", + "integrity": "sha512-4470Xi3GAPAjZqFcljX2xzckv1qeKPizoNkiS0+O4IoPR2ZNpcjE0pkhdihlDouK+x6QOast26B4Q/O9DJnwSg==", "dev": true, "requires": { + "call-bind": "^1.0.0", "define-properties": "^1.1.3", - "es-abstract": "^1.17.0-next.1" + "es-abstract": "^1.18.0-next.1" } }, "arrify": { @@ -1031,15 +990,15 @@ "integrity": "sha512-NfJ4UzBCcQGLDlQq7nHxH+tv3kyZ0hHQqF5BO6J7tNJeP5do1llPr8dZ8zHonfhAu0PHAdMkSo+8o0wxg9lZWw==" }, "assertion-error": { - "version": "1.0.0", - "resolved": "https://registry.npmjs.org/assertion-error/-/assertion-error-1.0.0.tgz", - "integrity": "sha512-g/gZV+G476cnmtYI+Ko9d5khxSoCSoom/EaNmmCfwpOvBXEJ18qwFrxfP1/CsIqk2no1sAKKwxndV0tP7ROOFQ==", + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/assertion-error/-/assertion-error-1.1.0.tgz", + "integrity": "sha512-jgsaNduz+ndvGyFt3uSuWqvy4lCnIJiovtouQN5JZHOKCS2QuhEdbcQHFhVksz2N2U9hXJo8odG7ETyWlEeuDw==", "dev": true }, "astral-regex": { - "version": "1.0.0", - "resolved": "https://registry.npmjs.org/astral-regex/-/astral-regex-1.0.0.tgz", - "integrity": "sha512-+Ryf6g3BKoRc7jfp7ad8tM4TtMiaWvbF/1/sQcZPkkS7ag3D5nMBCe2UfOTONtAkaG0tO0ij3C5Lwmf1EiyjHg==", + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/astral-regex/-/astral-regex-2.0.0.tgz", + "integrity": "sha512-Z7tMw1ytTXt5jqMcOP+OQteU1VuNK9Y02uuJtKQ1Sv69jXQKKg5cibLwGJow8yzZP+eAc18EmLGPal0bp36rvQ==", "dev": true }, "async": { @@ -1147,12 +1106,6 @@ "type-is": "~1.6.17" } }, - "boolify": { - "version": "1.0.1", - "resolved": "https://registry.npmjs.org/boolify/-/boolify-1.0.1.tgz", - "integrity": "sha512-ma2q0Tc760dW54CdOyJjhrg/a54317o1zYADQJFgperNGKIKgAUGIcKnuMiff8z57+yGlrGNEt4lPgZfCgTJgA==", - "dev": true - }, "brace-expansion": { "version": "1.1.11", "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-1.1.11.tgz", @@ -1208,6 +1161,16 @@ "resolved": "https://registry.npmjs.org/bytes/-/bytes-3.1.0.tgz", "integrity": "sha512-zauLjrfCG+xvoyaqLoV8bLVXXNGC4JqlxFCutSDWA6fJrTo2ZuvLYTqZ7aHBLZSMOopbzwv8f+wZcVzfVTI2Dg==" }, + "call-bind": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/call-bind/-/call-bind-1.0.2.tgz", + "integrity": "sha512-7O+FbCihrB5WGbFYesctwmTKae6rOiIzmz1icreWJ+0aA7LJfuqhEso2T9ncpcFtzMQtzXf2QGGueWJGTYsqrA==", + "dev": true, + "requires": { + "function-bind": "^1.1.1", + "get-intrinsic": "^1.0.2" + } + }, "callsites": { "version": "3.1.0", "resolved": "https://registry.npmjs.org/callsites/-/callsites-3.1.0.tgz", @@ -1215,35 +1178,37 @@ "dev": true }, "camelcase": { - "version": "5.3.1", - "resolved": "https://registry.npmjs.org/camelcase/-/camelcase-5.3.1.tgz", - "integrity": "sha512-L28STB170nwWS63UjtlEOE3dldQApaJXZkOI1uMFfzf3rRuPegHaHesyee+YxQ+W6SvRDQV6UrdOdRiR153wJg==", + "version": "6.2.0", + "resolved": "https://registry.npmjs.org/camelcase/-/camelcase-6.2.0.tgz", + "integrity": "sha512-c7wVvbw3f37nuobQNtgsgG9POC9qMbNuMQmTCqZv23b6MIz0fcYpBiOlv9gEN/hdLdnZTDQhg6e9Dq5M1vKvfg==", "dev": true }, - "camelcase-keys": { - "version": "6.2.2", - "resolved": "https://registry.npmjs.org/camelcase-keys/-/camelcase-keys-6.2.2.tgz", - "integrity": "sha512-YrwaA0vEKazPBkn0ipTiMpSajYDSe+KjQfrjhcBMxJt/znbvlHd8Pw/Vamaz5EB4Wfhs3SUR3Z9mwRu/P3s3Yg==", - "dev": true, - "requires": { - "camelcase": "^5.3.1", - "map-obj": "^4.0.0", - "quick-lru": "^4.0.1" - } - }, "caseless": { "version": "0.12.0", "resolved": "https://registry.npmjs.org/caseless/-/caseless-0.12.0.tgz", "integrity": "sha512-4tYFyifaFfGacoiObjJegolkwSU4xQNGbVgUiNYVUxbQ2x2lUsFvY4hVgVzGiIe6WLOPqycWXA40l+PWsxthUw==" }, "chai": { - "version": "1.9.2", - "resolved": "https://registry.npmjs.org/chai/-/chai-1.9.2.tgz", - "integrity": "sha512-olRoaitftnzWHFEAza6MXR4w+FfZrOVyV7r7U/Z8ObJefCgL8IuWkAuASJjSXrpP9wvgoL8+1dB9RbMLc2FkNg==", + "version": "4.3.4", + "resolved": "https://registry.npmjs.org/chai/-/chai-4.3.4.tgz", + "integrity": "sha512-yS5H68VYOCtN1cjfwumDSuzn/9c+yza4f3reKXlE5rUg7SFcCEy90gJvydNgOYtblyf4Zi6jIWRnXOgErta0KA==", "dev": true, "requires": { - "assertion-error": "1.0.0", - "deep-eql": "0.1.3" + "assertion-error": "^1.1.0", + "check-error": "^1.0.2", + "deep-eql": "^3.0.1", + "get-func-name": "^2.0.0", + "pathval": "^1.1.1", + "type-detect": "^4.0.5" + } + }, + "chai-as-promised": { + "version": "7.1.1", + "resolved": "https://registry.npmjs.org/chai-as-promised/-/chai-as-promised-7.1.1.tgz", + "integrity": "sha512-azL6xMoi+uxu6z4rhWQ1jbdUhOMhis2PvscD/xjLqNMkv3BPPp2JyyuTHOrf9BOosGpNQ11v6BKv/g57RXbiaA==", + "dev": true, + "requires": { + "check-error": "^1.0.2" } }, "chalk": { @@ -1257,17 +1222,17 @@ "supports-color": "^5.3.0" } }, - "chardet": { - "version": "0.7.0", - "resolved": "https://registry.npmjs.org/chardet/-/chardet-0.7.0.tgz", - "integrity": "sha512-mT8iDcrh03qDGRRmoA2hmBJnxpllMR+0/0qlzjqZES6NdiWDcZkCNAk4rPFZ9Q85r27unkiNNg8ZOiwZXBHwcA==", - "dev": true - }, "charenc": { "version": "0.0.2", "resolved": "https://registry.npmjs.org/charenc/-/charenc-0.0.2.tgz", "integrity": "sha1-wKHS86cJLgN3S/qD8UwPxXkKhmc=" }, + "check-error": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/check-error/-/check-error-1.0.2.tgz", + "integrity": "sha1-V00xLt2Iu13YkS6Sht1sCu1KrII=", + "dev": true + }, "chokidar": { "version": "3.5.1", "resolved": "https://registry.npmjs.org/chokidar/-/chokidar-3.5.1.tgz", @@ -1276,7 +1241,6 @@ "requires": { "anymatch": "~3.1.1", "braces": "~3.0.2", - "fsevents": "~2.3.1", "glob-parent": "~5.1.0", "is-binary-path": "~2.1.0", "is-glob": "~4.0.1", @@ -1289,55 +1253,15 @@ "resolved": "https://registry.npmjs.org/chownr/-/chownr-1.1.4.tgz", "integrity": "sha512-jJ0bqzaylmJtVnNgzTeSOs8DPavpbYgEr/b0YL8/2GO3xJEhInFmhKMUnEJQjZumK7KXGFhUy89PrsJWlakBVg==" }, - "cli-cursor": { - "version": "3.1.0", - "resolved": "https://registry.npmjs.org/cli-cursor/-/cli-cursor-3.1.0.tgz", - "integrity": "sha512-I/zHAwsKf9FqGoXM4WWRACob9+SNukZTd94DWF57E4toouRulbCxcUh6RKUEOQlYTHJnzkPMySvPNaaSLNfLZw==", - "dev": true, - "requires": { - "restore-cursor": "^3.1.0" - } - }, - "cli-width": { - "version": "2.2.1", - "resolved": "https://registry.npmjs.org/cli-width/-/cli-width-2.2.1.tgz", - "integrity": "sha512-GRMWDxpOB6Dgk2E5Uo+3eEBvtOOlimMmpbFiKuLFnQzYDavtLFY3K5ona41jgN/WdRZtG7utuVSVTL4HbZHGkw==", - "dev": true - }, "cliui": { - "version": "5.0.0", - "resolved": "https://registry.npmjs.org/cliui/-/cliui-5.0.0.tgz", - "integrity": "sha512-PYeGSEmmHM6zvoef2w8TPzlrnNpXIjTipYK780YswmIP9vjxmd6Y2a3CB2Ks6/AU8NHjZugXvo8w3oWM2qnwXA==", + "version": "7.0.4", + "resolved": "https://registry.npmjs.org/cliui/-/cliui-7.0.4.tgz", + "integrity": "sha512-OcRE68cOsVMXp1Yvonl/fzkQOyjLSu/8bhPDfQt0e0/Eb283TKP20Fs2MqoPsr9SwA595rRCA+QMzYc9nBP+JQ==", "dev": true, "requires": { - "string-width": "^3.1.0", - "strip-ansi": "^5.2.0", - "wrap-ansi": "^5.1.0" - }, - "dependencies": { - "emoji-regex": { - "version": "7.0.3", - "resolved": "https://registry.npmjs.org/emoji-regex/-/emoji-regex-7.0.3.tgz", - "integrity": "sha512-CwBLREIQ7LvYFB0WyRvwhq5N5qPhc6PMjD6bYggFlI5YyDgl+0vxq5VHbMOFqLg7hfWzmu8T5Z1QofhmTIhItA==", - "dev": true - }, - "is-fullwidth-code-point": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/is-fullwidth-code-point/-/is-fullwidth-code-point-2.0.0.tgz", - "integrity": "sha512-VHskAKYM8RfSFXwee5t5cbN5PZeq1Wrh6qd5bkyiXIf6UQcN6w/A0eXM9r6t8d+GYOh+o6ZhiEnb88LN/Y8m2w==", - "dev": true - }, - "string-width": { - "version": "3.1.0", - "resolved": "https://registry.npmjs.org/string-width/-/string-width-3.1.0.tgz", - "integrity": "sha512-vafcv6KjVZKSgz06oM/H6GDBrAtz8vdhQakGjFIvNrHA6y3HCF1CInLy+QLq8dTJPQ1b+KDUqDFctkdRW44e1w==", - "dev": true, - "requires": { - "emoji-regex": "^7.0.1", - "is-fullwidth-code-point": "^2.0.0", - "strip-ansi": "^5.1.0" - } - } + "string-width": "^4.2.0", + "strip-ansi": "^6.0.0", + "wrap-ansi": "^7.0.0" } }, "cluster-key-slot": { @@ -1383,12 +1307,6 @@ "resolved": "https://registry.npmjs.org/commander/-/commander-2.1.0.tgz", "integrity": "sha1-0SG7roYNmZKj1Re6lvVliOR8Z4E=" }, - "common-tags": { - "version": "1.8.0", - "resolved": "https://registry.npmjs.org/common-tags/-/common-tags-1.8.0.tgz", - "integrity": "sha512-6P6g0uetGpW/sdyUy/iQQCbFF0kWVMSIVSyYz7Zgjcgh8mgw8PQzDNZeyZ5DQ2gM7LBoZPHmnjz8rUthkBG5tw==", - "dev": true - }, "compressible": { "version": "2.0.18", "resolved": "https://registry.npmjs.org/compressible/-/compressible-2.0.18.tgz", @@ -1464,12 +1382,6 @@ "resolved": "https://registry.npmjs.org/console-log-level/-/console-log-level-1.4.1.tgz", "integrity": "sha512-VZzbIORbP+PPcN/gg3DXClTLPLg5Slwd5fL2MIc+o1qZ4BXBvWyc6QxPk6T/Mkr6IVjRpoAGf32XxP3ZWMVRcQ==" }, - "contains-path": { - "version": "0.1.0", - "resolved": "https://registry.npmjs.org/contains-path/-/contains-path-0.1.0.tgz", - "integrity": "sha512-OKZnPGeMQy2RPaUIBPFFd71iNf4791H12MCRuVQDnzGRwCYNYmTDy5pdafo2SLAcEMKzTOQnLWG4QdcjeJUMEg==", - "dev": true - }, "content-disposition": { "version": "0.5.3", "resolved": "https://registry.npmjs.org/content-disposition/-/content-disposition-0.5.3.tgz", @@ -1531,36 +1443,20 @@ "integrity": "sha512-Alvs19Vgq07eunykd3Xy2jF0/qSNv2u7KDbAek9H5liV1UMijbqFs5cycZvv5dVsvseT/U4H8/7/w8Koh35C4A==", "dev": true }, - "core-js": { - "version": "3.6.5", - "resolved": "https://registry.npmjs.org/core-js/-/core-js-3.6.5.tgz", - "integrity": "sha512-vZVEEwZoIsI+vPEuoF9Iqf5H7/M3eeQqWlQnYa8FSKKePuYTf5MWnxb5SDAzCa60b3JBRS5g9b+Dq7b1y/RCrA==", - "dev": true - }, "core-util-is": { "version": "1.0.2", "resolved": "https://registry.npmjs.org/core-util-is/-/core-util-is-1.0.2.tgz", "integrity": "sha512-3lqz5YjWTYnW6dlDa5TLaTCcShfar1e40rmcJVwCBJC6mWlFuj0eCHIElmG1g5kyuJ/GD+8Wn4FFCcz4gJPfaQ==" }, "cross-spawn": { - "version": "6.0.5", - "resolved": "https://registry.npmjs.org/cross-spawn/-/cross-spawn-6.0.5.tgz", - "integrity": "sha512-eTVLrBSt7fjbDygz805pMnstIs2VTBNkRm0qxZd+M7A5XDdxVRWO5MxGBXZhjY4cqLYLdtrGqRf8mBPmzwSpWQ==", + "version": "7.0.3", + "resolved": "https://registry.npmjs.org/cross-spawn/-/cross-spawn-7.0.3.tgz", + "integrity": "sha512-iRDPJKUPVEND7dHPO8rkbOnPpyDygcDFtWjpeWNCgy8WP2rXcxXL8TskReQl6OrB2G7+UJrags1q15Fudc7G6w==", "dev": true, "requires": { - "nice-try": "^1.0.4", - "path-key": "^2.0.1", - "semver": "^5.5.0", - "shebang-command": "^1.2.0", - "which": "^1.2.9" - }, - "dependencies": { - "semver": { - "version": "5.7.1", - "resolved": "https://registry.npmjs.org/semver/-/semver-5.7.1.tgz", - "integrity": "sha512-sauaDf/PZdVgrLTNYHRtpXa1iRiKcaebiKQ1BJdpQlWH2lCvexQdX55snPFyK7QzpudqbCI0qXFfOasHdyNDGQ==", - "dev": true - } + "path-key": "^3.1.0", + "shebang-command": "^2.0.0", + "which": "^2.0.1" } }, "crypt": { @@ -1590,18 +1486,18 @@ } }, "decamelize": { - "version": "1.2.0", - "resolved": "https://registry.npmjs.org/decamelize/-/decamelize-1.2.0.tgz", - "integrity": "sha512-z2S+W9X73hAUUki+N+9Za2lBlun89zigOyGrsax+KUQ6wKW4ZoWpEYBkGhQjwAjjDCkWxhY0VKEhk8wzY7F5cA==", + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/decamelize/-/decamelize-4.0.0.tgz", + "integrity": "sha512-9iE1PgSik9HeIIw2JO94IidnE3eBoQrFJ3w7sFuzSX4DpmZ3v5sZpUiV5Swcf6mQEF+Y0ru8Neo+p+nyh2J+hQ==", "dev": true }, "deep-eql": { - "version": "0.1.3", - "resolved": "https://registry.npmjs.org/deep-eql/-/deep-eql-0.1.3.tgz", - "integrity": "sha512-6sEotTRGBFiNcqVoeHwnfopbSpi5NbH1VWJmYCVkmxMmaVTT0bUTrNaGyBwhgP4MZL012W/mkzIn3Da+iDYweg==", + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/deep-eql/-/deep-eql-3.0.1.tgz", + "integrity": "sha512-+QeIQyN5ZuO+3Uk5DYh6/1eKO0m0YmJFGNmFHGACpf1ClL1nmlV/p4gNgbl2pJGxgXb4faqo6UE+M5ACEMyVcw==", "dev": true, "requires": { - "type-detect": "0.1.1" + "type-detect": "^4.0.0" } }, "deep-extend": { @@ -1665,12 +1561,6 @@ "integrity": "sha512-58lmxKSA4BNyLz+HHMUzlOEpg09FV+ev6ZMe3vJihgdxzgcwZ8VoEEPmALCZG9LmqfVoNMMKpttIYTVG6uDY7A==", "dev": true }, - "dlv": { - "version": "1.1.3", - "resolved": "https://registry.npmjs.org/dlv/-/dlv-1.1.3.tgz", - "integrity": "sha512-+HlytyjlPKnIG8XuRG8WvmBP8xs8P71y+SKKS6ZXWoEgLuePxtDoUEiH7WkdePWrQ5JBpE6aoVqfZfJUQkjXwA==", - "dev": true - }, "doctrine": { "version": "3.0.0", "resolved": "https://registry.npmjs.org/doctrine/-/doctrine-3.0.0.tgz", @@ -1757,6 +1647,15 @@ "once": "^1.4.0" } }, + "enquirer": { + "version": "2.3.6", + "resolved": "https://registry.npmjs.org/enquirer/-/enquirer-2.3.6.tgz", + "integrity": "sha512-yjNnPr315/FjS4zIsUxYguYUPP2e1NK4d7E7ZOLiyYCcbFBiTMyID+2wvm2w6+pZ/odMA7cRkjhsPbltwBOrLg==", + "dev": true, + "requires": { + "ansi-colors": "^4.1.1" + } + }, "ent": { "version": "2.2.0", "resolved": "https://registry.npmjs.org/ent/-/ent-2.2.0.tgz", @@ -1772,22 +1671,27 @@ } }, "es-abstract": { - "version": "1.17.6", - "resolved": "https://registry.npmjs.org/es-abstract/-/es-abstract-1.17.6.tgz", - "integrity": "sha512-Fr89bON3WFyUi5EvAeI48QTWX0AyekGgLA8H+c+7fbfCkJwRWRMLd8CQedNEyJuoYYhmtEqY92pgte1FAhBlhw==", + "version": "1.18.3", + "resolved": "https://registry.npmjs.org/es-abstract/-/es-abstract-1.18.3.tgz", + "integrity": "sha512-nQIr12dxV7SSxE6r6f1l3DtAeEYdsGpps13dR0TwJg1S8gyp4ZPgy3FZcHBgbiQqnoqSTb+oC+kO4UQ0C/J8vw==", "dev": true, "requires": { + "call-bind": "^1.0.2", "es-to-primitive": "^1.2.1", "function-bind": "^1.1.1", + "get-intrinsic": "^1.1.1", "has": "^1.0.3", - "has-symbols": "^1.0.1", - "is-callable": "^1.2.0", - "is-regex": "^1.1.0", - "object-inspect": "^1.7.0", + "has-symbols": "^1.0.2", + "is-callable": "^1.2.3", + "is-negative-zero": "^2.0.1", + "is-regex": "^1.1.3", + "is-string": "^1.0.6", + "object-inspect": "^1.10.3", "object-keys": "^1.1.1", - "object.assign": "^4.1.0", - "string.prototype.trimend": "^1.0.1", - "string.prototype.trimstart": "^1.0.1" + "object.assign": "^4.1.2", + "string.prototype.trimend": "^1.0.4", + "string.prototype.trimstart": "^1.0.4", + "unbox-primitive": "^1.0.1" } }, "es-to-primitive": { @@ -1815,61 +1719,168 @@ "escape-string-regexp": { "version": "1.0.5", "resolved": "https://registry.npmjs.org/escape-string-regexp/-/escape-string-regexp-1.0.5.tgz", - "integrity": "sha512-vbRorB5FUQWvla16U8R/qgaFIya2qGzwDrNmCZuYKrbdSUMG6I1ZCGQRefkRVhuOkIGVne7BQ35DSfo1qvJqFg==", + "integrity": "sha1-G2HAViGQqN/2rjuyzwIAyhMLhtQ=", "dev": true }, "eslint": { - "version": "6.8.0", - "resolved": "https://registry.npmjs.org/eslint/-/eslint-6.8.0.tgz", - "integrity": "sha512-K+Iayyo2LtyYhDSYwz5D5QdWw0hCacNzyq1Y821Xna2xSJj7cijoLLYmLxTQgcgZ9mC61nryMy9S7GRbYpI5Ig==", + "version": "7.30.0", + "resolved": "https://registry.npmjs.org/eslint/-/eslint-7.30.0.tgz", + "integrity": "sha512-VLqz80i3as3NdloY44BQSJpFw534L9Oh+6zJOUaViV4JPd+DaHwutqP7tcpkW3YiXbK6s05RZl7yl7cQn+lijg==", "dev": true, "requires": { - "@babel/code-frame": "^7.0.0", + "@babel/code-frame": "7.12.11", + "@eslint/eslintrc": "^0.4.2", + "@humanwhocodes/config-array": "^0.5.0", "ajv": "^6.10.0", - "chalk": "^2.1.0", - "cross-spawn": "^6.0.5", + "chalk": "^4.0.0", + "cross-spawn": "^7.0.2", "debug": "^4.0.1", "doctrine": "^3.0.0", - "eslint-scope": "^5.0.0", - "eslint-utils": "^1.4.3", - "eslint-visitor-keys": "^1.1.0", - "espree": "^6.1.2", - "esquery": "^1.0.1", + "enquirer": "^2.3.5", + "escape-string-regexp": "^4.0.0", + "eslint-scope": "^5.1.1", + "eslint-utils": "^2.1.0", + "eslint-visitor-keys": "^2.0.0", + "espree": "^7.3.1", + "esquery": "^1.4.0", "esutils": "^2.0.2", - "file-entry-cache": "^5.0.1", + "fast-deep-equal": "^3.1.3", + "file-entry-cache": "^6.0.1", "functional-red-black-tree": "^1.0.1", - "glob-parent": "^5.0.0", - "globals": "^12.1.0", + "glob-parent": "^5.1.2", + "globals": "^13.6.0", "ignore": "^4.0.6", "import-fresh": "^3.0.0", "imurmurhash": "^0.1.4", - "inquirer": "^7.0.0", "is-glob": "^4.0.0", "js-yaml": "^3.13.1", "json-stable-stringify-without-jsonify": "^1.0.1", - "levn": "^0.3.0", - "lodash": "^4.17.14", + "levn": "^0.4.1", + "lodash.merge": "^4.6.2", "minimatch": "^3.0.4", - "mkdirp": "^0.5.1", "natural-compare": "^1.4.0", - "optionator": "^0.8.3", + "optionator": "^0.9.1", "progress": "^2.0.0", - "regexpp": "^2.0.1", - "semver": "^6.1.2", - "strip-ansi": "^5.2.0", - "strip-json-comments": "^3.0.1", - "table": "^5.2.3", + "regexpp": "^3.1.0", + "semver": "^7.2.1", + "strip-ansi": "^6.0.0", + "strip-json-comments": "^3.1.0", + "table": "^6.0.9", "text-table": "^0.2.0", "v8-compile-cache": "^2.0.3" }, "dependencies": { - "debug": { + "chalk": { "version": "4.1.1", - "resolved": "https://registry.npmjs.org/debug/-/debug-4.1.1.tgz", - "integrity": "sha512-pYAIzeRo8J6KPEaJ0VWOh5Pzkbw/RetuzehGM7QRRX5he4fPHx2rdKMB256ehJCkX+XRQm16eZLqLNS8RSZXZw==", + "resolved": "https://registry.npmjs.org/chalk/-/chalk-4.1.1.tgz", + "integrity": "sha512-diHzdDKxcU+bAsUboHLPEDQiw0qEe0qd7SYUn3HgcFlWgbDcfLGswOHYeGrHKzG9z6UYf01d9VFMfZxPM1xZSg==", "dev": true, "requires": { - "ms": "^2.1.1" + "ansi-styles": "^4.1.0", + "supports-color": "^7.1.0" + }, + "dependencies": { + "ansi-styles": { + "version": "4.3.0", + "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-4.3.0.tgz", + "integrity": "sha512-zbB9rCJAT1rbjiVDb2hqKFHNYLxgtk8NURxZ3IZwD3F6NtxbXZQCnnSi1Lkx+IDohdPlFp222wVALIheZJQSEg==", + "dev": true, + "requires": { + "color-convert": "^2.0.1" + } + }, + "supports-color": { + "version": "7.2.0", + "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-7.2.0.tgz", + "integrity": "sha512-qpCAvRl9stuOHveKsn7HncJRvv501qIacKzQlO/+Lwxc9+0q2wLyv4Dfvt80/DPn2pqOBsJdDiogXGR9+OvwRw==", + "dev": true, + "requires": { + "has-flag": "^4.0.0" + } + } + } + }, + "color-convert": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/color-convert/-/color-convert-2.0.1.tgz", + "integrity": "sha512-RRECPsj7iu/xb5oKYcsFHSppFNnsj/52OVTRKb4zP5onXwVF3zVmmToNcOfGC+CRDpfK/U584fMg38ZHCaElKQ==", + "dev": true, + "requires": { + "color-name": "~1.1.4" + } + }, + "color-name": { + "version": "1.1.4", + "resolved": "https://registry.npmjs.org/color-name/-/color-name-1.1.4.tgz", + "integrity": "sha512-dOy+3AuW3a2wNbZHIuMZpTcgjGuLU/uBL/ubcZF9OXbDo8ff4O8yVp5Bf0efS8uEoYo5q4Fx7dY9OgQGXgAsQA==", + "dev": true + }, + "debug": { + "version": "4.3.2", + "resolved": "https://registry.npmjs.org/debug/-/debug-4.3.2.tgz", + "integrity": "sha512-mOp8wKcvj7XxC78zLgw/ZA+6TSgkoE2C/ienthhRD298T7UNwAg9diBpLRxC0mOezLl4B0xV7M0cCO6P/O0Xhw==", + "dev": true, + "requires": { + "ms": "2.1.2" + } + }, + "escape-string-regexp": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/escape-string-regexp/-/escape-string-regexp-4.0.0.tgz", + "integrity": "sha512-TtpcNJ3XAzx3Gq8sWRzJaVajRs0uVxA2YAkdb1jm2YkPz4G6egUFAyA3n5vtEIZefPk5Wa4UXbKuS5fKkJWdgA==", + "dev": true + }, + "eslint-visitor-keys": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/eslint-visitor-keys/-/eslint-visitor-keys-2.1.0.tgz", + "integrity": "sha512-0rSmRBzXgDzIsD6mGdJgevzgezI534Cer5L/vyMX0kHzT/jiB43jRhd9YUlMGYLQy2zprNmoT8qasCGtY+QaKw==", + "dev": true + }, + "estraverse": { + "version": "5.2.0", + "resolved": "https://registry.npmjs.org/estraverse/-/estraverse-5.2.0.tgz", + "integrity": "sha512-BxbNGGNm0RyRYvUdHpIwv9IWzeM9XClbOxwoATuFdOE7ZE6wHL+HQ5T8hoPM+zHvmKzzsEqhgy0GrQ5X13afiQ==" + }, + "glob": { + "version": "7.1.7", + "resolved": "https://registry.npmjs.org/glob/-/glob-7.1.7.tgz", + "integrity": "sha512-OvD9ENzPLbegENnYP5UUfJIirTg4+XwMWGaQfQTY0JenxNvvIKP3U3/tAQSPIu/lHxXYSZmpXlUHeqAIdKzBLQ==", + "requires": { + "fs.realpath": "^1.0.0", + "inflight": "^1.0.4", + "inherits": "2", + "minimatch": "^3.0.4", + "once": "^1.3.0", + "path-is-absolute": "^1.0.0" + } + }, + "glob-parent": { + "version": "5.1.2", + "resolved": "https://registry.npmjs.org/glob-parent/-/glob-parent-5.1.2.tgz", + "integrity": "sha512-AOIgSQCepiJYwP3ARnGx+5VnTu2HBYdzbGP45eLw1vr3zB3vZLeyed1sC9hnbcOc9/SrMyM5RPQrkGz4aS9Zow==", + "dev": true, + "requires": { + "is-glob": "^4.0.1" + } + }, + "has-flag": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/has-flag/-/has-flag-4.0.0.tgz", + "integrity": "sha512-EykJT/Q1KjTWctppgIAgfSO0tKVuZUjhgMr17kqTumMl6Afv3EISleU7qZUzoXDFTAHTDC4NOoG/ZxU3EvlMPQ==", + "dev": true + }, + "json-schema-traverse": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/json-schema-traverse/-/json-schema-traverse-1.0.0.tgz", + "integrity": "sha512-NM8/P9n3XjXhIZn1lLhkFaACTOURQXjWhV4BA/RnOv8xvgqtqpAX9IO4mRQxSx1Rlo4tqzeqb0sOlruaOy3dug==" + }, + "lru-cache": { + "version": "6.0.0", + "resolved": "https://registry.npmjs.org/lru-cache/-/lru-cache-6.0.0.tgz", + "integrity": "sha512-Jo6dJ04CmSjuznwJSS3pUeWmd/H0ffTlkXXgwZi+eq1UCmqQwCh+eLsYOYCwY991i2Fah4h1BEMCx4qThGbsiA==", + "dev": true, + "requires": { + "yallist": "^4.0.0" } }, "ms": { @@ -1877,22 +1888,42 @@ "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.2.tgz", "integrity": "sha512-sGkPx+VjMtmA6MX27oA4FBFELFCZZ4S4XqeGOXCv68tT+jb3vk/RyaKWP0PTKyWtmLSM0b+adUTEvbs1PEaH2w==", "dev": true + }, + "rimraf": { + "version": "3.0.2", + "resolved": "https://registry.npmjs.org/rimraf/-/rimraf-3.0.2.tgz", + "integrity": "sha512-JZkJMZkAGFFPP2YqXZXPbMlMBgsxzE8ILs4lMIX/2o0L9UBw9O/Y3o6wFw/i9YLapcUJWwqbi3kdxIPdC62TIA==", + "requires": { + "glob": "^7.1.3" + } + }, + "semver": { + "version": "7.3.5", + "resolved": "https://registry.npmjs.org/semver/-/semver-7.3.5.tgz", + "integrity": "sha512-PoeGJYh8HK4BTO/a9Tf6ZG3veo/A7ZVsYrSA6J8ny9nb3B1VrpkuN+z9OE5wfE5p6H4LchYZsegiQgbJD94ZFQ==", + "dev": true, + "requires": { + "lru-cache": "^6.0.0" + } + }, + "yallist": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/yallist/-/yallist-4.0.0.tgz", + "integrity": "sha512-3wdGidZyq5PB084XLES5TpOSRA3wjXAlIWMhum2kRcv/41Sn2emQ0dycQW4uZXLejwKvg6EsvbdlVL+FYEct7A==", + "dev": true } } }, "eslint-config-prettier": { - "version": "6.11.0", - "resolved": "https://registry.npmjs.org/eslint-config-prettier/-/eslint-config-prettier-6.11.0.tgz", - "integrity": "sha512-oB8cpLWSAjOVFEJhhyMZh6NOEOtBVziaqdDQ86+qhDHFbZXoRTM7pNSvFRfW/W/L/LrQ38C99J5CGuRBBzBsdA==", - "dev": true, - "requires": { - "get-stdin": "^6.0.0" - } + "version": "8.3.0", + "resolved": "https://registry.npmjs.org/eslint-config-prettier/-/eslint-config-prettier-8.3.0.tgz", + "integrity": "sha512-BgZuLUSeKzvlL/VUjx/Yb787VQ26RU3gGjA3iiFvdsp/2bMfVIWUVP7tjxtjS0e+HP409cPlPvNkQloz8C91ew==", + "dev": true }, "eslint-config-standard": { - "version": "14.1.1", - "resolved": "https://registry.npmjs.org/eslint-config-standard/-/eslint-config-standard-14.1.1.tgz", - "integrity": "sha512-Z9B+VR+JIXRxz21udPTL9HpFMyoMUEeX1G251EQ6e05WD9aPVtVBn09XUmZ259wCMlCDmYDSZG62Hhm+ZTJcUg==", + "version": "16.0.3", + "resolved": "https://registry.npmjs.org/eslint-config-standard/-/eslint-config-standard-16.0.3.tgz", + "integrity": "sha512-x4fmJL5hGqNJKGHSjnLdgA6U6h1YW/G2dW9fA+cyVur4SK6lyue8+UgNKWlZtUDTXvgKDD/Oa3GQjmB5kjtVvg==", "dev": true }, "eslint-import-resolver-node": { @@ -1906,25 +1937,42 @@ } }, "eslint-module-utils": { - "version": "2.6.0", - "resolved": "https://registry.npmjs.org/eslint-module-utils/-/eslint-module-utils-2.6.0.tgz", - "integrity": "sha512-6j9xxegbqe8/kZY8cYpcp0xhbK0EgJlg3g9mib3/miLaExuuwc3n5UEfSnU6hWMbT0FAYVvDbL9RrRgpUeQIvA==", + "version": "2.6.1", + "resolved": "https://registry.npmjs.org/eslint-module-utils/-/eslint-module-utils-2.6.1.tgz", + "integrity": "sha512-ZXI9B8cxAJIH4nfkhTwcRTEAnrVfobYqwjWy/QMCZ8rHkZHFjf9yO4BzpiF9kCSfNlMG54eKigISHpX0+AaT4A==", "dev": true, "requires": { - "debug": "^2.6.9", + "debug": "^3.2.7", "pkg-dir": "^2.0.0" + }, + "dependencies": { + "debug": { + "version": "3.2.7", + "resolved": "https://registry.npmjs.org/debug/-/debug-3.2.7.tgz", + "integrity": "sha512-CFjzYYAi4ThfiQvizrFQevTTXHtnCqWfe7x1AhgEscTz6ZbLbfoLRLPugTQyBth6f8ZERVUSyWHFD/7Wu4t1XQ==", + "dev": true, + "requires": { + "ms": "^2.1.1" + } + }, + "ms": { + "version": "2.1.3", + "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.3.tgz", + "integrity": "sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA==", + "dev": true + } } }, "eslint-plugin-chai-expect": { - "version": "2.1.0", - "resolved": "https://registry.npmjs.org/eslint-plugin-chai-expect/-/eslint-plugin-chai-expect-2.1.0.tgz", - "integrity": "sha512-rd0/4mjMV6c3i0o4DKkWI4uaFN9DK707kW+/fDphaDI6HVgxXnhML9Xgt5vHnTXmSSnDhupuCFBgsEAEpchXmQ==", + "version": "2.2.0", + "resolved": "https://registry.npmjs.org/eslint-plugin-chai-expect/-/eslint-plugin-chai-expect-2.2.0.tgz", + "integrity": "sha512-ExTJKhgeYMfY8wDj3UiZmgpMKJOUHGNHmWMlxT49JUDB1vTnw0sSNfXJSxnX+LcebyBD/gudXzjzD136WqPJrQ==", "dev": true }, "eslint-plugin-chai-friendly": { - "version": "0.5.0", - "resolved": "https://registry.npmjs.org/eslint-plugin-chai-friendly/-/eslint-plugin-chai-friendly-0.5.0.tgz", - "integrity": "sha512-Pxe6z8C9fP0pn2X2nGFU/b3GBOCM/5FVus1hsMwJsXP3R7RiXFl7g0ksJbsc0GxiLyidTW4mEFk77qsNn7Tk7g==", + "version": "0.6.0", + "resolved": "https://registry.npmjs.org/eslint-plugin-chai-friendly/-/eslint-plugin-chai-friendly-0.6.0.tgz", + "integrity": "sha512-Uvvv1gkbRGp/qfN15B0kQyQWg+oFA8buDSqrwmW3egNSk/FpqH2MjQqKOuKwmEL6w4QIQrIjDp+gg6kGGmD3oQ==", "dev": true }, "eslint-plugin-es": { @@ -1935,86 +1983,60 @@ "requires": { "eslint-utils": "^2.0.0", "regexpp": "^3.0.0" - }, - "dependencies": { - "eslint-utils": { - "version": "2.1.0", - "resolved": "https://registry.npmjs.org/eslint-utils/-/eslint-utils-2.1.0.tgz", - "integrity": "sha512-w94dQYoauyvlDc43XnGB8lU3Zt713vNChgt4EWwhXAP2XkBvndfxF0AgIqKOOasjPIPzj9JqgwkwbCYD0/V3Zg==", - "dev": true, - "requires": { - "eslint-visitor-keys": "^1.1.0" - } - }, - "regexpp": { - "version": "3.1.0", - "resolved": "https://registry.npmjs.org/regexpp/-/regexpp-3.1.0.tgz", - "integrity": "sha512-ZOIzd8yVsQQA7j8GCSlPGXwg5PfmA1mrq0JP4nGhh54LaKN3xdai/vHUDu74pKwV8OxseMS65u2NImosQcSD0Q==", - "dev": true - } } }, "eslint-plugin-import": { - "version": "2.21.2", - "resolved": "https://registry.npmjs.org/eslint-plugin-import/-/eslint-plugin-import-2.21.2.tgz", - "integrity": "sha512-FEmxeGI6yaz+SnEB6YgNHlQK1Bs2DKLM+YF+vuTk5H8J9CLbJLtlPvRFgZZ2+sXiKAlN5dpdlrWOjK8ZoZJpQA==", + "version": "2.23.4", + "resolved": "https://registry.npmjs.org/eslint-plugin-import/-/eslint-plugin-import-2.23.4.tgz", + "integrity": "sha512-6/wP8zZRsnQFiR3iaPFgh5ImVRM1WN5NUWfTIRqwOdeiGJlBcSk82o1FEVq8yXmy4lkIzTo7YhHCIxlU/2HyEQ==", "dev": true, "requires": { - "array-includes": "^3.1.1", - "array.prototype.flat": "^1.2.3", - "contains-path": "^0.1.0", + "array-includes": "^3.1.3", + "array.prototype.flat": "^1.2.4", "debug": "^2.6.9", - "doctrine": "1.5.0", - "eslint-import-resolver-node": "^0.3.3", - "eslint-module-utils": "^2.6.0", + "doctrine": "^2.1.0", + "eslint-import-resolver-node": "^0.3.4", + "eslint-module-utils": "^2.6.1", + "find-up": "^2.0.0", "has": "^1.0.3", + "is-core-module": "^2.4.0", "minimatch": "^3.0.4", - "object.values": "^1.1.1", - "read-pkg-up": "^2.0.0", - "resolve": "^1.17.0", + "object.values": "^1.1.3", + "pkg-up": "^2.0.0", + "read-pkg-up": "^3.0.0", + "resolve": "^1.20.0", "tsconfig-paths": "^3.9.0" }, "dependencies": { "doctrine": { - "version": "1.5.0", - "resolved": "https://registry.npmjs.org/doctrine/-/doctrine-1.5.0.tgz", - "integrity": "sha512-lsGyRuYr4/PIB0txi+Fy2xOMI2dGaTguCaotzFGkVZuKR5usKfcRWIFKNM3QNrU7hh/+w2bwTW+ZeXPK5l8uVg==", + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/doctrine/-/doctrine-2.1.0.tgz", + "integrity": "sha512-35mSku4ZXK0vfCuHEDAwt55dg2jNajHZ1odvF+8SSr82EsZY4QmXfuWso8oEd8zRhVObSN18aM0CjSdoBX7zIw==", "dev": true, "requires": { - "esutils": "^2.0.2", - "isarray": "^1.0.0" + "esutils": "^2.0.2" } }, "resolve": { - "version": "1.17.0", - "resolved": "https://registry.npmjs.org/resolve/-/resolve-1.17.0.tgz", - "integrity": "sha512-ic+7JYiV8Vi2yzQGFWOkiZD5Z9z7O2Zhm9XMaTxdJExKasieFCr+yXZ/WmXsckHiKl12ar0y6XiXDx3m4RHn1w==", + "version": "1.20.0", + "resolved": "https://registry.npmjs.org/resolve/-/resolve-1.20.0.tgz", + "integrity": "sha512-wENBPt4ySzg4ybFQW2TT1zMQucPK95HSh/nq2CFTZVOGut2+pQvSsgtda4d26YrYcr067wjbmzOG8byDPBX63A==", "dev": true, "requires": { + "is-core-module": "^2.2.0", "path-parse": "^1.0.6" } } } }, "eslint-plugin-mocha": { - "version": "6.3.0", - "resolved": "https://registry.npmjs.org/eslint-plugin-mocha/-/eslint-plugin-mocha-6.3.0.tgz", - "integrity": "sha512-Cd2roo8caAyG21oKaaNTj7cqeYRWW1I2B5SfpKRp0Ip1gkfwoR1Ow0IGlPWnNjzywdF4n+kHL8/9vM6zCJUxdg==", + "version": "8.2.0", + "resolved": "https://registry.npmjs.org/eslint-plugin-mocha/-/eslint-plugin-mocha-8.2.0.tgz", + "integrity": "sha512-8oOR47Ejt+YJPNQzedbiklDqS1zurEaNrxXpRs+Uk4DMDPVmKNagShFeUaYsfvWP55AhI+P1non5QZAHV6K78A==", "dev": true, "requires": { - "eslint-utils": "^2.0.0", - "ramda": "^0.27.0" - }, - "dependencies": { - "eslint-utils": { - "version": "2.1.0", - "resolved": "https://registry.npmjs.org/eslint-utils/-/eslint-utils-2.1.0.tgz", - "integrity": "sha512-w94dQYoauyvlDc43XnGB8lU3Zt713vNChgt4EWwhXAP2XkBvndfxF0AgIqKOOasjPIPzj9JqgwkwbCYD0/V3Zg==", - "dev": true, - "requires": { - "eslint-visitor-keys": "^1.1.0" - } - } + "eslint-utils": "^2.1.0", + "ramda": "^0.27.1" } }, "eslint-plugin-node": { @@ -2031,15 +2053,6 @@ "semver": "^6.1.0" }, "dependencies": { - "eslint-utils": { - "version": "2.1.0", - "resolved": "https://registry.npmjs.org/eslint-utils/-/eslint-utils-2.1.0.tgz", - "integrity": "sha512-w94dQYoauyvlDc43XnGB8lU3Zt713vNChgt4EWwhXAP2XkBvndfxF0AgIqKOOasjPIPzj9JqgwkwbCYD0/V3Zg==", - "dev": true, - "requires": { - "eslint-visitor-keys": "^1.1.0" - } - }, "ignore": { "version": "5.1.8", "resolved": "https://registry.npmjs.org/ignore/-/ignore-5.1.8.tgz", @@ -2063,26 +2076,20 @@ "integrity": "sha512-VoM09vT7bfA7D+upt+FjeBO5eHIJQBUWki1aPvB+vbNiHS3+oGIJGIeyBtKQTME6UPXXy3vV07OL1tHd3ANuDw==", "dev": true }, - "eslint-plugin-standard": { - "version": "4.0.1", - "resolved": "https://registry.npmjs.org/eslint-plugin-standard/-/eslint-plugin-standard-4.0.1.tgz", - "integrity": "sha512-v/KBnfyaOMPmZc/dmc6ozOdWqekGp7bBGq4jLAecEfPGmfKiWS4sA8sC0LqiV9w5qmXAtXVn4M3p1jSyhY85SQ==", - "dev": true - }, "eslint-scope": { - "version": "5.1.0", - "resolved": "https://registry.npmjs.org/eslint-scope/-/eslint-scope-5.1.0.tgz", - "integrity": "sha512-iiGRvtxWqgtx5m8EyQUJihBloE4EnYeGE/bz1wSPwJE6tZuJUtHlhqDM4Xj2ukE8Dyy1+HCZ4hE0fzIVMzb58w==", + "version": "5.1.1", + "resolved": "https://registry.npmjs.org/eslint-scope/-/eslint-scope-5.1.1.tgz", + "integrity": "sha512-2NxwbF/hZ0KpepYN0cNbo+FN6XoK7GaHlQhgx/hIZl6Va0bF45RQOOwhLIy8lQDbuCiadSLCBnH2CFYquit5bw==", "dev": true, "requires": { - "esrecurse": "^4.1.0", + "esrecurse": "^4.3.0", "estraverse": "^4.1.1" } }, "eslint-utils": { - "version": "1.4.3", - "resolved": "https://registry.npmjs.org/eslint-utils/-/eslint-utils-1.4.3.tgz", - "integrity": "sha512-fbBN5W2xdY45KulGXmLHZ3c3FHfVYmKg0IrAKGOkT/464PQsx2UeIzfz1RmEci+KLm1bBaAzZAh8+/E+XAeZ8Q==", + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/eslint-utils/-/eslint-utils-2.1.0.tgz", + "integrity": "sha512-w94dQYoauyvlDc43XnGB8lU3Zt713vNChgt4EWwhXAP2XkBvndfxF0AgIqKOOasjPIPzj9JqgwkwbCYD0/V3Zg==", "dev": true, "requires": { "eslint-visitor-keys": "^1.1.0" @@ -2095,20 +2102,20 @@ "dev": true }, "espree": { - "version": "6.2.1", - "resolved": "https://registry.npmjs.org/espree/-/espree-6.2.1.tgz", - "integrity": "sha512-ysCxRQY3WaXJz9tdbWOwuWr5Y/XrPTGX9Kiz3yoUXwW0VZ4w30HTkQLaGx/+ttFjF8i+ACbArnB4ce68a9m5hw==", + "version": "7.3.1", + "resolved": "https://registry.npmjs.org/espree/-/espree-7.3.1.tgz", + "integrity": "sha512-v3JCNCE64umkFpmkFGqzVKsOT0tN1Zr+ueqLZfpV1Ob8e+CEgPWa+OxCoGH3tnhimMKIaBm4m/vaRpJ/krRz2g==", "dev": true, "requires": { - "acorn": "^7.1.1", - "acorn-jsx": "^5.2.0", - "eslint-visitor-keys": "^1.1.0" + "acorn": "^7.4.0", + "acorn-jsx": "^5.3.1", + "eslint-visitor-keys": "^1.3.0" }, "dependencies": { "acorn": { - "version": "7.3.1", - "resolved": "https://registry.npmjs.org/acorn/-/acorn-7.3.1.tgz", - "integrity": "sha512-tLc0wSnatxAQHVHUapaHdz72pi9KUyHjq5KyHjGg9Y8Ifdc79pTh2XvI6I1/chZbnM7QtNKzh66ooDogPZSleA==", + "version": "7.4.1", + "resolved": "https://registry.npmjs.org/acorn/-/acorn-7.4.1.tgz", + "integrity": "sha512-nQyp0o1/mNdbTO1PO6kHkwSrmgZ0MT/jCCpNiwbUjGoRN4dlBhqJtoQuCnEOKzgTVwg0ZWiCoQy6SxMebQVh8A==", "dev": true } } @@ -2120,31 +2127,28 @@ "dev": true }, "esquery": { - "version": "1.3.1", - "resolved": "https://registry.npmjs.org/esquery/-/esquery-1.3.1.tgz", - "integrity": "sha512-olpvt9QG0vniUBZspVRN6lwB7hOZoTRtT+jzR+tS4ffYx2mzbw+z0XCOk44aaLYKApNX5nMm+E+P6o25ip/DHQ==", + "version": "1.4.0", + "resolved": "https://registry.npmjs.org/esquery/-/esquery-1.4.0.tgz", + "integrity": "sha512-cCDispWt5vHHtwMY2YrAQ4ibFkAL8RbH5YGBnZBc90MolvvfkkQcJro/aZiAQUlQ3qgrYS6D6v8Gc5G5CQsc9w==", + "dev": true + }, + "esrecurse": { + "version": "4.3.0", + "resolved": "https://registry.npmjs.org/esrecurse/-/esrecurse-4.3.0.tgz", + "integrity": "sha512-KmfKL3b6G+RXvP8N1vr3Tq1kL/oCFgn2NYXEtqP8/L3pKapUA4G8cFVaoF3SU323CD4XypR/ffioHmkti6/Tag==", "dev": true, "requires": { - "estraverse": "^5.1.0" + "estraverse": "^5.2.0" }, "dependencies": { "estraverse": { - "version": "5.1.0", - "resolved": "https://registry.npmjs.org/estraverse/-/estraverse-5.1.0.tgz", - "integrity": "sha512-FyohXK+R0vE+y1nHLoBM7ZTyqRpqAlhdZHCWIWEviFLiGB8b04H6bQs8G+XTthacvT8VuwvteiP7RJSxMs8UEw==", + "version": "5.2.0", + "resolved": "https://registry.npmjs.org/estraverse/-/estraverse-5.2.0.tgz", + "integrity": "sha512-BxbNGGNm0RyRYvUdHpIwv9IWzeM9XClbOxwoATuFdOE7ZE6wHL+HQ5T8hoPM+zHvmKzzsEqhgy0GrQ5X13afiQ==", "dev": true } } }, - "esrecurse": { - "version": "4.2.1", - "resolved": "https://registry.npmjs.org/esrecurse/-/esrecurse-4.2.1.tgz", - "integrity": "sha512-64RBB++fIOAXPw3P9cy89qfMlvZEXZkqqJkjqqXIvzP5ezRZjW+lPWjw35UX/3EhUPFYbg5ER4JYgDw4007/DQ==", - "dev": true, - "requires": { - "estraverse": "^4.1.0" - } - }, "estraverse": { "version": "4.3.0", "resolved": "https://registry.npmjs.org/estraverse/-/estraverse-4.3.0.tgz", @@ -2279,17 +2283,6 @@ "resolved": "https://registry.npmjs.org/extend/-/extend-3.0.2.tgz", "integrity": "sha512-fjquC59cD7CyW6urNXK0FBufkZcoiGG80wTuPujX590cB5Ttln20E2UB4S/WARVqhXffZl2LNgS+gQdPIIim/g==" }, - "external-editor": { - "version": "3.1.0", - "resolved": "https://registry.npmjs.org/external-editor/-/external-editor-3.1.0.tgz", - "integrity": "sha512-hMQ4CX1p1izmuLYyZqLMO/qGNw10wSv9QDCPfzXfyFrOaCSSoRfqE1Kf1s5an66J5JZC62NewG+mK49jOCtQew==", - "dev": true, - "requires": { - "chardet": "^0.7.0", - "iconv-lite": "^0.4.24", - "tmp": "^0.0.33" - } - }, "extsprintf": { "version": "1.3.0", "resolved": "https://registry.npmjs.org/extsprintf/-/extsprintf-1.3.0.tgz", @@ -2322,22 +2315,13 @@ "resolved": "https://registry.npmjs.org/fast-text-encoding/-/fast-text-encoding-1.0.1.tgz", "integrity": "sha512-x4FEgaz3zNRtJfLFqJmHWxkMDDvXVtaznj2V9jiP8ACUJrUgist4bP9FmDL2Vew2Y9mEQI/tG4GqabaitYp9CQ==" }, - "figures": { - "version": "3.2.0", - "resolved": "https://registry.npmjs.org/figures/-/figures-3.2.0.tgz", - "integrity": "sha512-yaduQFRKLXYOGgEn6AZau90j3ggSOyiqXU0F9JZfeXYhNa+Jk4X+s45A2zg5jns87GAFa34BBm2kXw4XpNcbdg==", - "dev": true, - "requires": { - "escape-string-regexp": "^1.0.5" - } - }, "file-entry-cache": { - "version": "5.0.1", - "resolved": "https://registry.npmjs.org/file-entry-cache/-/file-entry-cache-5.0.1.tgz", - "integrity": "sha512-bCg29ictuBaKUwwArK4ouCaqDgLZcysCFLmM/Yn/FDoqndh/9vNuQfXRDvTuXKLxfD/JtZQGKFT8MGcJBK644g==", + "version": "6.0.1", + "resolved": "https://registry.npmjs.org/file-entry-cache/-/file-entry-cache-6.0.1.tgz", + "integrity": "sha512-7Gps/XWymbLk2QLYK4NzpMOrYjMhdIxXuIvy2QBsLE6ljuodKvdkWs/cpyJJ3CVIVpH0Oi1Hvg1ovbMzLdFBBg==", "dev": true, "requires": { - "flat-cache": "^2.0.1" + "flat-cache": "^3.0.4" } }, "file-uri-to-path": { @@ -2371,7 +2355,7 @@ "find-up": { "version": "2.1.0", "resolved": "https://registry.npmjs.org/find-up/-/find-up-2.1.0.tgz", - "integrity": "sha512-NWzkk0jSJtTt08+FBFMvXoeZnOJD+jTtsRmBYbAIzJdX6l7dLgR7CTubCM5/eDdPUBvLCeVasP1brfVR/9/EZQ==", + "integrity": "sha1-RdG35QbHF93UgndaK3eSCjwMV6c=", "dev": true, "requires": { "locate-path": "^2.0.0" @@ -2389,45 +2373,18 @@ "dev": true }, "flat-cache": { - "version": "2.0.1", - "resolved": "https://registry.npmjs.org/flat-cache/-/flat-cache-2.0.1.tgz", - "integrity": "sha512-LoQe6yDuUMDzQAEH8sgmh4Md6oZnc/7PjtwjNFSzveXqSHt6ka9fPBuso7IGf9Rz4uqnSnWiFH2B/zj24a5ReA==", + "version": "3.0.4", + "resolved": "https://registry.npmjs.org/flat-cache/-/flat-cache-3.0.4.tgz", + "integrity": "sha512-dm9s5Pw7Jc0GvMYbshN6zchCA9RgQlzzEZX3vylR9IqFfS8XciblUXOKfW6SiuJ0e13eDYZoZV5wdrev7P3Nwg==", "dev": true, "requires": { - "flatted": "^2.0.0", - "rimraf": "2.6.3", - "write": "1.0.3" - }, - "dependencies": { - "glob": { - "version": "7.1.6", - "resolved": "https://registry.npmjs.org/glob/-/glob-7.1.6.tgz", - "integrity": "sha512-LwaxwyZ72Lk7vZINtNNrywX0ZuLyStrdDtabefZKAY5ZGJhVtgdznluResxNmPitE0SAO+O26sWTHeKSI2wMBA==", - "dev": true, - "requires": { - "fs.realpath": "^1.0.0", - "inflight": "^1.0.4", - "inherits": "2", - "minimatch": "^3.0.4", - "once": "^1.3.0", - "path-is-absolute": "^1.0.0" - } - }, - "rimraf": { - "version": "2.6.3", - "resolved": "https://registry.npmjs.org/rimraf/-/rimraf-2.6.3.tgz", - "integrity": "sha512-mwqeW5XsA2qAejG46gYdENaxXjx9onRNCfn7L0duuP4hCuTIi/QO7PDK07KJfp1d+izWPrzEJDcSqBa0OZQriA==", - "dev": true, - "requires": { - "glob": "^7.1.3" - } - } + "flatted": "^3.1.0" } }, "flatted": { - "version": "2.0.2", - "resolved": "https://registry.npmjs.org/flatted/-/flatted-2.0.2.tgz", - "integrity": "sha512-r5wGx7YeOwNWNlCA0wQ86zKyDLMQr+/RB8xy74M4hTphfmjlijTSSXGuH8rnvKZnfT9i+75zmd8jcKdMR4O6jA==", + "version": "3.2.1", + "resolved": "https://registry.npmjs.org/flatted/-/flatted-3.2.1.tgz", + "integrity": "sha512-OMQjaErSFHmHqZe+PSidH5n8j3O0F2DdnVh8JB4j4eUQ2k6KvB0qGfrKIhapvez5JerBbmWkaLYUYWISaESoXg==", "dev": true }, "forever-agent": { @@ -2468,13 +2425,6 @@ "resolved": "https://registry.npmjs.org/fs.realpath/-/fs.realpath-1.0.0.tgz", "integrity": "sha512-OO0pH2lK6a0hZnAdau5ItzHPI6pUlvI7jMVnxUQRtw4owF2wk8lOSabtGDCTP4Ggrg2MbGnWO9X8K1t4+fGMDw==" }, - "fsevents": { - "version": "2.3.2", - "resolved": "https://registry.npmjs.org/fsevents/-/fsevents-2.3.2.tgz", - "integrity": "sha512-xiqMQR4xAeHTuB9uWm+fFRcIOgKBMiOBP+eXiyT7jsgVCq1bkVygt00oASowB7EdtpOHaaPgKt812P9ab+DDKA==", - "dev": true, - "optional": true - }, "function-bind": { "version": "1.1.1", "resolved": "https://registry.npmjs.org/function-bind/-/function-bind-1.1.1.tgz", @@ -2577,12 +2527,23 @@ "integrity": "sha512-DyFP3BM/3YHTQOCUL/w0OZHR0lpKeGrxotcHWcqNEdnltqFwXVfhEBQ94eIo34AfQpo0rGki4cyIiftY06h2Fg==", "dev": true }, - "get-stdin": { - "version": "6.0.0", - "resolved": "https://registry.npmjs.org/get-stdin/-/get-stdin-6.0.0.tgz", - "integrity": "sha512-jp4tHawyV7+fkkSKyvjuLZswblUtz+SQKzSWnBbii16BuZksJlU1wuBYXY75r+duh/llF1ur6oNwi+2ZzjKZ7g==", + "get-func-name": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/get-func-name/-/get-func-name-2.0.0.tgz", + "integrity": "sha1-6td0q+5y4gQJQzoGY2YCPdaIekE=", "dev": true }, + "get-intrinsic": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/get-intrinsic/-/get-intrinsic-1.1.1.tgz", + "integrity": "sha512-kWZrnVM42QCiEA2Ig1bG8zjoIMOgxWwYCEeNdwY6Tv/cOSeGpcoX4pXHfKUxNKVoArnrEr2e9srnAxxGIraS9Q==", + "dev": true, + "requires": { + "function-bind": "^1.1.1", + "has": "^1.0.3", + "has-symbols": "^1.0.1" + } + }, "getpass": { "version": "0.1.7", "resolved": "https://registry.npmjs.org/getpass/-/getpass-0.1.7.tgz", @@ -2614,12 +2575,12 @@ } }, "globals": { - "version": "12.4.0", - "resolved": "https://registry.npmjs.org/globals/-/globals-12.4.0.tgz", - "integrity": "sha512-BWICuzzDvDoH54NHKCseDanAhE3CeDorgDL5MT6LMXXj2WCnd9UC2szdk4AWLfjdgNBCXLUanXYcpBBKOSWGwg==", + "version": "13.10.0", + "resolved": "https://registry.npmjs.org/globals/-/globals-13.10.0.tgz", + "integrity": "sha512-piHC3blgLGFjvOuMmWZX60f+na1lXFDhQXBf1UYp2fXPXqvEUbOhNwi6BsQ0bQishwedgnjkwv1d9zKf+MWw3g==", "dev": true, "requires": { - "type-fest": "^0.8.1" + "type-fest": "^0.20.2" } }, "google-auth-library": { @@ -2764,9 +2725,9 @@ } }, "graceful-fs": { - "version": "4.2.4", - "resolved": "https://registry.npmjs.org/graceful-fs/-/graceful-fs-4.2.4.tgz", - "integrity": "sha512-WjKPNJF79dtJAVniUlGGWHYGz2jWxT6VhN/4m1NdkbZ2nOsEF+cI1Edgql5zCRhs/VsQYRvrXctxktVXZUkixw==", + "version": "4.2.6", + "resolved": "https://registry.npmjs.org/graceful-fs/-/graceful-fs-4.2.6.tgz", + "integrity": "sha512-nTnJ528pbqxYanhpDYsi4Rd8MAeaBA67+RZ10CM1m3bTAVFEDcd5AuA4a6W5YkGZ1iNXHzZz8T6TBKLeBuNriQ==", "dev": true }, "growl": { @@ -2808,22 +2769,11 @@ "function-bind": "^1.1.1" } }, - "has-ansi": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/has-ansi/-/has-ansi-2.0.0.tgz", - "integrity": "sha512-C8vBJ8DwUCx19vhm7urhTuUsr4/IyP6l4VzNQDv+ryHQObW3TTTp9yB68WpYgRe2bbaGuZ/se74IqFeVnMnLZg==", - "dev": true, - "requires": { - "ansi-regex": "^2.0.0" - }, - "dependencies": { - "ansi-regex": { - "version": "2.1.1", - "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-2.1.1.tgz", - "integrity": "sha512-TIGnTpdo+E3+pCyAluZvtED5p5wCqLdezCyhPZzKPcxvFplEt4i+W7OONCKgeZFT3+y5NZZfOOS/Bdcanm1MYA==", - "dev": true - } - } + "has-bigints": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/has-bigints/-/has-bigints-1.0.1.tgz", + "integrity": "sha512-LSBS2LjbNBTf6287JEbEzvJgftkF5qFkmCo9hDRpAzKhUOlJ+hx8dd4USs00SgsUNwc4617J9ki5YtEClM2ffA==", + "dev": true }, "has-flag": { "version": "3.0.0", @@ -2832,9 +2782,9 @@ "dev": true }, "has-symbols": { - "version": "1.0.1", - "resolved": "https://registry.npmjs.org/has-symbols/-/has-symbols-1.0.1.tgz", - "integrity": "sha512-PLcsoqu++dmEIZB+6totNFKq/7Do+Z0u4oT0zKOJNl3lYK6vGwwu2hjHs+68OEZbTjiUE9bgOABXbP/GvrS0Kg==", + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/has-symbols/-/has-symbols-1.0.2.tgz", + "integrity": "sha512-chXa79rL/UC2KlX17jo3vRGz0azaWEx5tGqZg5pO3NUyEJVB17dMruQlzCCOfUvElghKcm5194+BCRvi2Rv/Gw==", "dev": true }, "has-unicode": { @@ -2968,12 +2918,6 @@ "integrity": "sha512-JmXMZ6wuvDmLiHEml9ykzqO6lwFbof0GG4IkcGaENdCRDDmMVnny7s5HsIgHCbaq0w2MyPhDqkhTUgS2LU2PHA==", "dev": true }, - "indent-string": { - "version": "4.0.0", - "resolved": "https://registry.npmjs.org/indent-string/-/indent-string-4.0.0.tgz", - "integrity": "sha512-EdDDZu4A2OyIK7Lr/2zG+w5jmbuk1DVBnEwREQvBzspBJkCEbRa8GxU1lghYcaGJCnRWibjDXlq779X1/y5xwg==", - "dev": true - }, "inflight": { "version": "1.0.6", "resolved": "https://registry.npmjs.org/inflight/-/inflight-1.0.6.tgz", @@ -2993,88 +2937,6 @@ "resolved": "https://registry.npmjs.org/ini/-/ini-1.3.8.tgz", "integrity": "sha512-JV/yugV2uzW5iMRSiZAyDtQd+nxtUnjeLt0acNdw98kKLrvuRVyB80tsREOE7yvGVgalhZ6RNXCmEHkUKBKxew==" }, - "inquirer": { - "version": "7.2.0", - "resolved": "https://registry.npmjs.org/inquirer/-/inquirer-7.2.0.tgz", - "integrity": "sha512-E0c4rPwr9ByePfNlTIB8z51kK1s2n6jrHuJeEHENl/sbq2G/S1auvibgEwNR4uSyiU+PiYHqSwsgGiXjG8p5ZQ==", - "dev": true, - "requires": { - "ansi-escapes": "^4.2.1", - "chalk": "^3.0.0", - "cli-cursor": "^3.1.0", - "cli-width": "^2.0.0", - "external-editor": "^3.0.3", - "figures": "^3.0.0", - "lodash": "^4.17.15", - "mute-stream": "0.0.8", - "run-async": "^2.4.0", - "rxjs": "^6.5.3", - "string-width": "^4.1.0", - "strip-ansi": "^6.0.0", - "through": "^2.3.6" - }, - "dependencies": { - "ansi-styles": { - "version": "4.2.1", - "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-4.2.1.tgz", - "integrity": "sha512-9VGjrMsG1vePxcSweQsN20KY/c4zN0h9fLjqAbwbPfahM3t+NL+M9HC8xeXG2I8pX5NoamTGNuomEUFI7fcUjA==", - "dev": true, - "requires": { - "@types/color-name": "^1.1.1", - "color-convert": "^2.0.1" - } - }, - "chalk": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/chalk/-/chalk-3.0.0.tgz", - "integrity": "sha512-4D3B6Wf41KOYRFdszmDqMCGq5VV/uMAB273JILmO+3jAlh8X4qDtdtgCR3fxtbLEMzSx22QdhnDcJvu2u1fVwg==", - "dev": true, - "requires": { - "ansi-styles": "^4.1.0", - "supports-color": "^7.1.0" - } - }, - "color-convert": { - "version": "2.0.1", - "resolved": "https://registry.npmjs.org/color-convert/-/color-convert-2.0.1.tgz", - "integrity": "sha512-RRECPsj7iu/xb5oKYcsFHSppFNnsj/52OVTRKb4zP5onXwVF3zVmmToNcOfGC+CRDpfK/U584fMg38ZHCaElKQ==", - "dev": true, - "requires": { - "color-name": "~1.1.4" - } - }, - "color-name": { - "version": "1.1.4", - "resolved": "https://registry.npmjs.org/color-name/-/color-name-1.1.4.tgz", - "integrity": "sha512-dOy+3AuW3a2wNbZHIuMZpTcgjGuLU/uBL/ubcZF9OXbDo8ff4O8yVp5Bf0efS8uEoYo5q4Fx7dY9OgQGXgAsQA==", - "dev": true - }, - "has-flag": { - "version": "4.0.0", - "resolved": "https://registry.npmjs.org/has-flag/-/has-flag-4.0.0.tgz", - "integrity": "sha512-EykJT/Q1KjTWctppgIAgfSO0tKVuZUjhgMr17kqTumMl6Afv3EISleU7qZUzoXDFTAHTDC4NOoG/ZxU3EvlMPQ==", - "dev": true - }, - "strip-ansi": { - "version": "6.0.0", - "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-6.0.0.tgz", - "integrity": "sha512-AuvKTrTfQNYNIctbR1K/YGTR1756GycPsg7b9bdV9Duqur4gv6aKqHXah67Z8ImS7WEz5QVcOtlfW2rZEugt6w==", - "dev": true, - "requires": { - "ansi-regex": "^5.0.0" - } - }, - "supports-color": { - "version": "7.1.0", - "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-7.1.0.tgz", - "integrity": "sha512-oRSIpR8pxT1Wr2FquTNnGet79b3BWljqOuoW/h4oBhxJ/HUbX5nX6JSruTkvXDCFMwDPvsaTTbvMLKZWSy0R5g==", - "dev": true, - "requires": { - "has-flag": "^4.0.0" - } - } - } - }, "ioredis": { "version": "4.17.3", "resolved": "https://registry.npmjs.org/ioredis/-/ioredis-4.17.3.tgz", @@ -3119,7 +2981,13 @@ "is-arrayish": { "version": "0.2.1", "resolved": "https://registry.npmjs.org/is-arrayish/-/is-arrayish-0.2.1.tgz", - "integrity": "sha512-zz06S8t0ozoDXMG+ube26zeCTNXcKIPJZJi8hBrF4idCLms4CG9QtK7qBl1boi5ODzFpjswb5JPmHCbMpjaYzg==", + "integrity": "sha1-d8mYQFJ6qOyxqLppe4BkWnqSap0=", + "dev": true + }, + "is-bigint": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/is-bigint/-/is-bigint-1.0.2.tgz", + "integrity": "sha512-0JV5+SOCQkIdzjBK9buARcV804Ddu7A0Qet6sHi3FimE9ne6m4BGQZfRn+NZiXbBk4F4XmHfDZIipLj9pX8dSA==", "dev": true }, "is-binary-path": { @@ -3131,21 +2999,39 @@ "binary-extensions": "^2.0.0" } }, + "is-boolean-object": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/is-boolean-object/-/is-boolean-object-1.1.1.tgz", + "integrity": "sha512-bXdQWkECBUIAcCkeH1unwJLIpZYaa5VvuygSyS/c2lf719mTKZDU5UdDRlpd01UjADgmW8RfqaP+mRaVPdr/Ng==", + "dev": true, + "requires": { + "call-bind": "^1.0.2" + } + }, "is-buffer": { "version": "1.1.6", "resolved": "https://registry.npmjs.org/is-buffer/-/is-buffer-1.1.6.tgz", "integrity": "sha512-NcdALwpXkTm5Zvvbk7owOUSvVvBKDgKP5/ewfXEznmQFfs4ZRmanOeKBTjRVjka3QFoN6XJ+9F3USqfHqTaU5w==" }, "is-callable": { - "version": "1.2.0", - "resolved": "https://registry.npmjs.org/is-callable/-/is-callable-1.2.0.tgz", - "integrity": "sha512-pyVD9AaGLxtg6srb2Ng6ynWJqkHU9bEM087AKck0w8QwDarTfNcpIYoU8x8Hv2Icm8u6kFJM18Dag8lyqGkviw==", + "version": "1.2.3", + "resolved": "https://registry.npmjs.org/is-callable/-/is-callable-1.2.3.tgz", + "integrity": "sha512-J1DcMe8UYTBSrKezuIUTUwjXsho29693unXM2YhJUTR2txK/eG47bvNa/wipPFmZFgr/N6f1GA66dv0mEyTIyQ==", "dev": true }, + "is-core-module": { + "version": "2.5.0", + "resolved": "https://registry.npmjs.org/is-core-module/-/is-core-module-2.5.0.tgz", + "integrity": "sha512-TXCMSDsEHMEEZ6eCA8rwRDbLu55MRGmrctljsBX/2v1d9/GzqHOxW5c5oPSgrUt2vBFXebu9rGqckXGPWOlYpg==", + "dev": true, + "requires": { + "has": "^1.0.3" + } + }, "is-date-object": { - "version": "1.0.2", - "resolved": "https://registry.npmjs.org/is-date-object/-/is-date-object-1.0.2.tgz", - "integrity": "sha512-USlDT524woQ08aoZFzh3/Z6ch9Y/EWXEHQ/AaRN0SkKq4t2Jw2R2339tSXmwuVoY7LLlBCbOIlx2myP/L5zk0g==", + "version": "1.0.4", + "resolved": "https://registry.npmjs.org/is-date-object/-/is-date-object-1.0.4.tgz", + "integrity": "sha512-/b4ZVsG7Z5XVtIxs/h9W8nvfLgSAyKYdtGWQLbqy6jA1icmgjf8WCoTKgeS4wy5tYaPePouzFMANbnj94c2Z+A==", "dev": true }, "is-extglob": { @@ -3169,12 +3055,24 @@ "is-extglob": "^2.1.1" } }, + "is-negative-zero": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/is-negative-zero/-/is-negative-zero-2.0.1.tgz", + "integrity": "sha512-2z6JzQvZRa9A2Y7xC6dQQm4FSTSTNWjKIYYTt4246eMTJmIo0Q+ZyOsU66X8lxK1AbB92dFeglPLrhwpeRKO6w==", + "dev": true + }, "is-number": { "version": "7.0.0", "resolved": "https://registry.npmjs.org/is-number/-/is-number-7.0.0.tgz", "integrity": "sha512-41Cifkg6e8TylSpdtTpeLVMqvSBEVzTttHvERD741+pnZ8ANv0004MRL43QKPDlK9cGvNp6NZWZUBlbGXYxxng==", "dev": true }, + "is-number-object": { + "version": "1.0.5", + "resolved": "https://registry.npmjs.org/is-number-object/-/is-number-object-1.0.5.tgz", + "integrity": "sha512-RU0lI/n95pMoUKu9v1BZP5MBcZuNSVJkMkAG2dJqC4z2GlkGUNeH68SuHuBKBD/XFe+LHZ+f9BKkLET60Niedw==", + "dev": true + }, "is-obj": { "version": "2.0.0", "resolved": "https://registry.npmjs.org/is-obj/-/is-obj-2.0.0.tgz", @@ -3187,12 +3085,13 @@ "dev": true }, "is-regex": { - "version": "1.1.0", - "resolved": "https://registry.npmjs.org/is-regex/-/is-regex-1.1.0.tgz", - "integrity": "sha512-iI97M8KTWID2la5uYXlkbSDQIg4F6o1sYboZKKTDpnDQMLtUL86zxhgDet3Q2SriaYsyGqZ6Mn2SjbRKeLHdqw==", + "version": "1.1.3", + "resolved": "https://registry.npmjs.org/is-regex/-/is-regex-1.1.3.tgz", + "integrity": "sha512-qSVXFz28HM7y+IWX6vLCsexdlvzT1PJNFSBuaQLQ5o0IEw8UDYW6/2+eCMVyIsbM8CNLX2a/QWmSpyxYEHY7CQ==", "dev": true, "requires": { - "has-symbols": "^1.0.1" + "call-bind": "^1.0.2", + "has-symbols": "^1.0.2" } }, "is-stream": { @@ -3206,18 +3105,18 @@ "integrity": "sha512-xj0XPvmr7bQFTvirqnFr50o0hQIh6ZItDqloxt5aJrR4NQsYeSsyFQERYGCAzfindAcnKjINnwEEgLx4IqVzQw==" }, "is-string": { - "version": "1.0.5", - "resolved": "https://registry.npmjs.org/is-string/-/is-string-1.0.5.tgz", - "integrity": "sha512-buY6VNRjhQMiF1qWDouloZlQbRhDPCebwxSjxMjxgemYT46YMd2NR0/H+fBhEfWX4A/w9TBJ+ol+okqJKFE6vQ==", + "version": "1.0.6", + "resolved": "https://registry.npmjs.org/is-string/-/is-string-1.0.6.tgz", + "integrity": "sha512-2gdzbKUuqtQ3lYNrUTQYoClPhm7oQu4UdpSZMp1/DGgkHBT8E2Z1l0yMdb6D4zNAxwDiMv8MdulKROJGNl0Q0w==", "dev": true }, "is-symbol": { - "version": "1.0.3", - "resolved": "https://registry.npmjs.org/is-symbol/-/is-symbol-1.0.3.tgz", - "integrity": "sha512-OwijhaRSgqvhm/0ZdAcXNZt9lYdKFpcRDT5ULUuYXPoT794UNOdU+gpT6Rzo7b4V2HUl/op6GqY894AZwv9faQ==", + "version": "1.0.4", + "resolved": "https://registry.npmjs.org/is-symbol/-/is-symbol-1.0.4.tgz", + "integrity": "sha512-C/CPBqKWnvdcxqIARxyOh4v1UUEOCHpgDa0WYgpKDFMszcrPcffg5uhwSgPCLD2WWxmq6isisz87tzT01tuGhg==", "dev": true, "requires": { - "has-symbols": "^1.0.1" + "has-symbols": "^1.0.2" } }, "is-typedarray": { @@ -3270,6 +3169,12 @@ "bignumber.js": "^7.0.0" } }, + "json-parse-better-errors": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/json-parse-better-errors/-/json-parse-better-errors-1.0.2.tgz", + "integrity": "sha512-mrqyZKfX5EhL7hvqcV6WG1yYjnjeuYDzDhhcAAUrq8Po85NBQBJP+ZDUT75qZQ98IkUoBqdkExkukOU7Ts2wrw==", + "dev": true + }, "json-schema": { "version": "0.2.3", "resolved": "https://registry.npmjs.org/json-schema/-/json-schema-0.2.3.tgz", @@ -3292,12 +3197,12 @@ "integrity": "sha512-ZClg6AaYvamvYEE82d3Iyd3vSSIjQ+odgjaTzRuO3s7toCdFKczob2i0zCh7JE8kWn17yvAWhUVxvqGwUalsRA==" }, "json5": { - "version": "1.0.1", - "resolved": "https://registry.npmjs.org/json5/-/json5-1.0.1.tgz", - "integrity": "sha512-aKS4WQjPenRxiQsC93MNfjx+nbF4PAdYzmd/1JIj8HYzqfbu86beTuNgXDzPknWk0n0uARlyewZo4s++ES36Ow==", + "version": "2.2.0", + "resolved": "https://registry.npmjs.org/json5/-/json5-2.2.0.tgz", + "integrity": "sha512-f+8cldu7X/y7RAJurMEJmdoKXGB/X550w2Nr3tTbezL6RwEE/iMcm+tZnXeoZtKuOq6ft8+CqzEkrIgx1fPoQA==", "dev": true, "requires": { - "minimist": "^1.2.0" + "minimist": "^1.2.5" }, "dependencies": { "minimist": { @@ -3345,31 +3250,31 @@ } }, "levn": { - "version": "0.3.0", - "resolved": "https://registry.npmjs.org/levn/-/levn-0.3.0.tgz", - "integrity": "sha512-0OO4y2iOHix2W6ujICbKIaEQXvFQHue65vUG3pb5EUomzPI90z9hsA1VsO/dbIIpC53J8gxM9Q4Oho0jrCM/yA==", + "version": "0.4.1", + "resolved": "https://registry.npmjs.org/levn/-/levn-0.4.1.tgz", + "integrity": "sha512-+bT2uH4E5LGE7h/n3evcS/sQlJXCpIp6ym8OWJ5eV6+67Dsql/LaaT7qJBAt2rzfoa/5QBGBhxDix1dMt2kQKQ==", "dev": true, "requires": { - "prelude-ls": "~1.1.2", - "type-check": "~0.3.2" + "prelude-ls": "^1.2.1", + "type-check": "~0.4.0" } }, "load-json-file": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/load-json-file/-/load-json-file-2.0.0.tgz", - "integrity": "sha512-3p6ZOGNbiX4CdvEd1VcE6yi78UrGNpjHO33noGwHCnT/o2fyllJDepsm8+mFFv/DvtwFHht5HIHSyOy5a+ChVQ==", + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/load-json-file/-/load-json-file-4.0.0.tgz", + "integrity": "sha1-L19Fq5HjMhYjT9U62rZo607AmTs=", "dev": true, "requires": { "graceful-fs": "^4.1.2", - "parse-json": "^2.2.0", - "pify": "^2.0.0", + "parse-json": "^4.0.0", + "pify": "^3.0.0", "strip-bom": "^3.0.0" }, "dependencies": { "pify": { - "version": "2.3.0", - "resolved": "https://registry.npmjs.org/pify/-/pify-2.3.0.tgz", - "integrity": "sha512-udgsAY+fTnvv7kI7aaxbqwWNb0AHiB0qBO89PZKPkoTmGOgdbrHDKD+0B2X4uTfJ/FT1R09r9gTsjUjNJotuog==", + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/pify/-/pify-3.0.0.tgz", + "integrity": "sha1-5aSs0sEB/fPZpNB/DbxNtJ3SgXY=", "dev": true } } @@ -3377,19 +3282,13 @@ "locate-path": { "version": "2.0.0", "resolved": "https://registry.npmjs.org/locate-path/-/locate-path-2.0.0.tgz", - "integrity": "sha512-NCI2kiDkyR7VeEKm27Kda/iQHyKJe1Bu0FlTbYp3CqJu+9IFe9bLyAjMxf5ZDDbEg+iMPzB5zYyUTSm8wVTKmA==", + "integrity": "sha1-K1aLJl7slExtnA3pw9u7ygNUzY4=", "dev": true, "requires": { "p-locate": "^2.0.0", "path-exists": "^3.0.0" } }, - "lodash": { - "version": "4.17.21", - "resolved": "https://registry.npmjs.org/lodash/-/lodash-4.17.21.tgz", - "integrity": "sha512-v2kDEe57lecTulaDIuNTPy3Ry4gLGJ6Z1O3vE1krgXZNrsQ+LFTGHVxVjcXPs17LhbZVGedAJv8XZ1tvj5FvSg==", - "dev": true - }, "lodash.at": { "version": "4.6.0", "resolved": "https://registry.npmjs.org/lodash.at/-/lodash.at-4.6.0.tgz", @@ -3400,6 +3299,12 @@ "resolved": "https://registry.npmjs.org/lodash.camelcase/-/lodash.camelcase-4.3.0.tgz", "integrity": "sha1-soqmKIorn8ZRA1x3EfZathkDMaY=" }, + "lodash.clonedeep": { + "version": "4.5.0", + "resolved": "https://registry.npmjs.org/lodash.clonedeep/-/lodash.clonedeep-4.5.0.tgz", + "integrity": "sha1-4j8/nE+Pvd6HJSnBBxhXoIblzO8=", + "dev": true + }, "lodash.defaults": { "version": "4.2.0", "resolved": "https://registry.npmjs.org/lodash.defaults/-/lodash.defaults-4.2.0.tgz", @@ -3421,22 +3326,16 @@ "resolved": "https://registry.npmjs.org/lodash.has/-/lodash.has-4.5.2.tgz", "integrity": "sha1-0Z9NwQlQWMzL4rDN9O4P5Ko3yGI=" }, - "lodash.memoize": { - "version": "4.1.2", - "resolved": "https://registry.npmjs.org/lodash.memoize/-/lodash.memoize-4.1.2.tgz", - "integrity": "sha512-t7j+NzmgnQzTAYXcsHYLgimltOV1MXHtlOWf6GjL9Kj8GK5FInw5JotxvbOs+IvV1/Dzo04/fCGfLVs7aXb4Ag==", - "dev": true - }, "lodash.merge": { "version": "4.6.2", "resolved": "https://registry.npmjs.org/lodash.merge/-/lodash.merge-4.6.2.tgz", "integrity": "sha512-0KpjqXRVvrYyCsX1swR/XTK0va6VQkQM6MNo7PqW77ByjAhoARA8EfrP1N4+KlKj8YS0ZUCtRT/YUuhyYDujIQ==", "dev": true }, - "lodash.unescape": { - "version": "4.0.1", - "resolved": "https://registry.npmjs.org/lodash.unescape/-/lodash.unescape-4.0.1.tgz", - "integrity": "sha512-DhhGRshNS1aX6s5YdBE3njCCouPgnG29ebyHvImlZzXZf2SHgt+J08DHgytTPnpywNbO1Y8mNUFyQuIDBq2JZg==", + "lodash.truncate": { + "version": "4.4.2", + "resolved": "https://registry.npmjs.org/lodash.truncate/-/lodash.truncate-4.4.2.tgz", + "integrity": "sha1-WjUNoLERO4N+z//VgSy+WNbq4ZM=", "dev": true }, "log-driver": { @@ -3524,64 +3423,6 @@ } } }, - "loglevel": { - "version": "1.6.8", - "resolved": "https://registry.npmjs.org/loglevel/-/loglevel-1.6.8.tgz", - "integrity": "sha512-bsU7+gc9AJ2SqpzxwU3+1fedl8zAntbtC5XYlt3s2j1hJcn2PsXSmgN8TaLG/J1/2mod4+cE/3vNL70/c1RNCA==", - "dev": true - }, - "loglevel-colored-level-prefix": { - "version": "1.0.0", - "resolved": "https://registry.npmjs.org/loglevel-colored-level-prefix/-/loglevel-colored-level-prefix-1.0.0.tgz", - "integrity": "sha512-u45Wcxxc+SdAlh4yeF/uKlC1SPUPCy0gullSNKXod5I4bmifzk+Q4lSLExNEVn19tGaJipbZ4V4jbFn79/6mVA==", - "dev": true, - "requires": { - "chalk": "^1.1.3", - "loglevel": "^1.4.1" - }, - "dependencies": { - "ansi-regex": { - "version": "2.1.1", - "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-2.1.1.tgz", - "integrity": "sha512-TIGnTpdo+E3+pCyAluZvtED5p5wCqLdezCyhPZzKPcxvFplEt4i+W7OONCKgeZFT3+y5NZZfOOS/Bdcanm1MYA==", - "dev": true - }, - "ansi-styles": { - "version": "2.2.1", - "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-2.2.1.tgz", - "integrity": "sha512-kmCevFghRiWM7HB5zTPULl4r9bVFSWjz62MhqizDGUrq2NWuNMQyuv4tHHoKJHs69M/MF64lEcHdYIocrdWQYA==", - "dev": true - }, - "chalk": { - "version": "1.1.3", - "resolved": "https://registry.npmjs.org/chalk/-/chalk-1.1.3.tgz", - "integrity": "sha512-U3lRVLMSlsCfjqYPbLyVv11M9CPW4I728d6TCKMAOJueEeB9/8o+eSsMnxPJD+Q+K909sdESg7C+tIkoH6on1A==", - "dev": true, - "requires": { - "ansi-styles": "^2.2.1", - "escape-string-regexp": "^1.0.2", - "has-ansi": "^2.0.0", - "strip-ansi": "^3.0.0", - "supports-color": "^2.0.0" - } - }, - "strip-ansi": { - "version": "3.0.1", - "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-3.0.1.tgz", - "integrity": "sha512-VhumSSbBqDTP8p2ZLKj40UjBCV4+v8bUSEpUb4KjRgWk9pbqGF4REFj6KEagidb2f/M6AzC0EmFyDNGaw9OCzg==", - "dev": true, - "requires": { - "ansi-regex": "^2.0.0" - } - }, - "supports-color": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-2.0.0.tgz", - "integrity": "sha512-KKNVtd6pCYgPIKU4cp2733HWYCpplQhddZLBUryaAHou723x+FRzQ5Df824Fj+IyyuiQTRoub4SnIFfIcrp70g==", - "dev": true - } - } - }, "long": { "version": "4.0.0", "resolved": "https://registry.npmjs.org/long/-/long-4.0.0.tgz", @@ -3595,24 +3436,6 @@ "yallist": "^3.0.2" } }, - "make-plural": { - "version": "4.3.0", - "resolved": "https://registry.npmjs.org/make-plural/-/make-plural-4.3.0.tgz", - "integrity": "sha512-xTYd4JVHpSCW+aqDof6w/MebaMVNTVYBZhbB/vi513xXdiPT92JMVCo0Jq8W2UZnzYRFeVbQiQ+I25l13JuKvA==", - "dev": true, - "requires": { - "minimist": "^1.2.0" - }, - "dependencies": { - "minimist": { - "version": "1.2.5", - "resolved": "https://registry.npmjs.org/minimist/-/minimist-1.2.5.tgz", - "integrity": "sha512-FM9nNUYrRBAELZQT3xeZQ7fmMOBg6nWNmJKTcgsJeaLstP/UODVpGsr5OhXhhXg6f+qtJ8uiZ+PUxkDWcgIXLw==", - "dev": true, - "optional": true - } - } - }, "map-obj": { "version": "4.1.0", "resolved": "https://registry.npmjs.org/map-obj/-/map-obj-4.1.0.tgz", @@ -3638,29 +3461,6 @@ "resolved": "https://registry.npmjs.org/merge-descriptors/-/merge-descriptors-1.0.1.tgz", "integrity": "sha512-cCi6g3/Zr1iqQi6ySbseM1Xvooa98N0w31jzUYrXPX2xqObmFGHJ0tQ5u74H3mVh7wLouTseZyYIq39g8cNp1w==" }, - "messageformat": { - "version": "2.3.0", - "resolved": "https://registry.npmjs.org/messageformat/-/messageformat-2.3.0.tgz", - "integrity": "sha512-uTzvsv0lTeQxYI2y1NPa1lItL5VRI8Gb93Y2K2ue5gBPyrbJxfDi/EYWxh2PKv5yO42AJeeqblS9MJSh/IEk4w==", - "dev": true, - "requires": { - "make-plural": "^4.3.0", - "messageformat-formatters": "^2.0.1", - "messageformat-parser": "^4.1.2" - } - }, - "messageformat-formatters": { - "version": "2.0.1", - "resolved": "https://registry.npmjs.org/messageformat-formatters/-/messageformat-formatters-2.0.1.tgz", - "integrity": "sha512-E/lQRXhtHwGuiQjI7qxkLp8AHbMD5r2217XNe/SREbBlSawe0lOqsFb7rflZJmlQFSULNLIqlcjjsCPlB3m3Mg==", - "dev": true - }, - "messageformat-parser": { - "version": "4.1.3", - "resolved": "https://registry.npmjs.org/messageformat-parser/-/messageformat-parser-4.1.3.tgz", - "integrity": "sha512-2fU3XDCanRqeOCkn7R5zW5VQHWf+T3hH65SzuqRvjatBK7r4uyFa5mEX+k6F9Bd04LVM5G4/BHBTUJsOdW7uyg==", - "dev": true - }, "methods": { "version": "1.1.2", "resolved": "https://registry.npmjs.org/methods/-/methods-1.1.2.tgz", @@ -3684,12 +3484,6 @@ "mime-db": "~1.33.0" } }, - "mimic-fn": { - "version": "2.1.0", - "resolved": "https://registry.npmjs.org/mimic-fn/-/mimic-fn-2.1.0.tgz", - "integrity": "sha512-OqbOk5oEQeAZ8WXWydlu9HJjz9WVdEIvamMCcXmuqUYjTknH/sqsWvhQ3vgwKFRR1HpjvNBKQ37nbJgYzGqGcg==", - "dev": true - }, "minimatch": { "version": "3.0.4", "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-3.0.4.tgz", @@ -3772,7 +3566,6 @@ "version": "4.3.0", "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-4.3.0.tgz", "integrity": "sha512-zbB9rCJAT1rbjiVDb2hqKFHNYLxgtk8NURxZ3IZwD3F6NtxbXZQCnnSi1Lkx+IDohdPlFp222wVALIheZJQSEg==", - "dev": true, "requires": { "color-convert": "^2.0.1" } @@ -3783,22 +3576,10 @@ "integrity": "sha512-8+9WqebbFzpX9OR+Wa6O29asIogeRMzcGtAINdpMHHyAg10f05aSFVBbcEqGf/PXw1EjAZ+q2/bEBg3DvurK3Q==", "dev": true }, - "cliui": { - "version": "7.0.4", - "resolved": "https://registry.npmjs.org/cliui/-/cliui-7.0.4.tgz", - "integrity": "sha512-OcRE68cOsVMXp1Yvonl/fzkQOyjLSu/8bhPDfQt0e0/Eb283TKP20Fs2MqoPsr9SwA595rRCA+QMzYc9nBP+JQ==", - "dev": true, - "requires": { - "string-width": "^4.2.0", - "strip-ansi": "^6.0.0", - "wrap-ansi": "^7.0.0" - } - }, "color-convert": { "version": "2.0.1", "resolved": "https://registry.npmjs.org/color-convert/-/color-convert-2.0.1.tgz", "integrity": "sha512-RRECPsj7iu/xb5oKYcsFHSppFNnsj/52OVTRKb4zP5onXwVF3zVmmToNcOfGC+CRDpfK/U584fMg38ZHCaElKQ==", - "dev": true, "requires": { "color-name": "~1.1.4" } @@ -3806,8 +3587,7 @@ "color-name": { "version": "1.1.4", "resolved": "https://registry.npmjs.org/color-name/-/color-name-1.1.4.tgz", - "integrity": "sha512-dOy+3AuW3a2wNbZHIuMZpTcgjGuLU/uBL/ubcZF9OXbDo8ff4O8yVp5Bf0efS8uEoYo5q4Fx7dY9OgQGXgAsQA==", - "dev": true + "integrity": "sha512-dOy+3AuW3a2wNbZHIuMZpTcgjGuLU/uBL/ubcZF9OXbDo8ff4O8yVp5Bf0efS8uEoYo5q4Fx7dY9OgQGXgAsQA==" }, "debug": { "version": "4.3.1", @@ -3892,15 +3672,6 @@ "integrity": "sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA==", "dev": true }, - "p-limit": { - "version": "3.1.0", - "resolved": "https://registry.npmjs.org/p-limit/-/p-limit-3.1.0.tgz", - "integrity": "sha512-TYOanM3wGwNGsZN2cVTYPArw454xnXj5qmWF1bEoAc4+cU/ol7GVh7odevjp1FNHduHc3KZMcFduxU5Xc6uJRQ==", - "dev": true, - "requires": { - "yocto-queue": "^0.1.0" - } - }, "p-locate": { "version": "5.0.0", "resolved": "https://registry.npmjs.org/p-locate/-/p-locate-5.0.0.tgz", @@ -3916,21 +3687,6 @@ "integrity": "sha512-ak9Qy5Q7jYb2Wwcey5Fpvg2KoAc/ZIhLSLOSBmRmygPsGwkVVt0fZa0qrtMz+m6tJTAHfZQ8FnmB4MG4LWy7/w==", "dev": true }, - "strip-ansi": { - "version": "6.0.0", - "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-6.0.0.tgz", - "integrity": "sha512-AuvKTrTfQNYNIctbR1K/YGTR1756GycPsg7b9bdV9Duqur4gv6aKqHXah67Z8ImS7WEz5QVcOtlfW2rZEugt6w==", - "dev": true, - "requires": { - "ansi-regex": "^5.0.0" - } - }, - "strip-json-comments": { - "version": "3.1.1", - "resolved": "https://registry.npmjs.org/strip-json-comments/-/strip-json-comments-3.1.1.tgz", - "integrity": "sha512-6fPc+R4ihwqP6N/aIv2f1gMH8lOVtWQHoqC4yK6oSDVVocumAsfCqjkXnqiYMhmMwS/mEHLp7Vehlt3ql6lEig==", - "dev": true - }, "supports-color": { "version": "8.1.1", "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-8.1.1.tgz", @@ -3939,47 +3695,6 @@ "requires": { "has-flag": "^4.0.0" } - }, - "which": { - "version": "2.0.2", - "resolved": "https://registry.npmjs.org/which/-/which-2.0.2.tgz", - "integrity": "sha512-BLI3Tl1TW3Pvl70l3yq3Y64i+awpwXqsGBYWkkqMtnbXgrMD+yj7rhW0kuEDxzJaYXGjEW5ogapKNMEKNMjibA==", - "dev": true, - "requires": { - "isexe": "^2.0.0" - } - }, - "wrap-ansi": { - "version": "7.0.0", - "resolved": "https://registry.npmjs.org/wrap-ansi/-/wrap-ansi-7.0.0.tgz", - "integrity": "sha512-YVGIj2kamLSTxw6NsZjoBxfSwsn0ycdesmc4p+Q21c5zPuZ1pl+NfxVdxPtdHvmNVOQ6XSYG4AUtyt/Fi7D16Q==", - "dev": true, - "requires": { - "ansi-styles": "^4.0.0", - "string-width": "^4.1.0", - "strip-ansi": "^6.0.0" - } - }, - "yargs": { - "version": "16.2.0", - "resolved": "https://registry.npmjs.org/yargs/-/yargs-16.2.0.tgz", - "integrity": "sha512-D1mvvtDG0L5ft/jGWkLpG1+m0eQxOfaBvTNELraWj22wSVUMWxZUvYgJYcKh6jGGIkJFhH4IZPQhR4TKpc8mBw==", - "dev": true, - "requires": { - "cliui": "^7.0.2", - "escalade": "^3.1.1", - "get-caller-file": "^2.0.5", - "require-directory": "^2.1.1", - "string-width": "^4.2.0", - "y18n": "^5.0.5", - "yargs-parser": "^20.2.2" - } - }, - "yargs-parser": { - "version": "20.2.4", - "resolved": "https://registry.npmjs.org/yargs-parser/-/yargs-parser-20.2.4.tgz", - "integrity": "sha512-WOkpgNhPTlE73h4VFAFsOnomJVaovO8VqLDzy5saChRBFQFBoMYirowyW+Q9HB4HFF4Z7VZTiG3iSzJJA29yRA==", - "dev": true } } }, @@ -3999,12 +3714,6 @@ "resolved": "https://registry.npmjs.org/ms/-/ms-2.0.0.tgz", "integrity": "sha512-Tpp60P6IUJDTuOq/5Z8cdskzJujfwqfOTkrwIwj7IRISpnkJnT6SyJ4PCPnGMoFjC9ddhal5KVIYtAt97ix05A==" }, - "mute-stream": { - "version": "0.0.8", - "resolved": "https://registry.npmjs.org/mute-stream/-/mute-stream-0.0.8.tgz", - "integrity": "sha512-nnbWWOkoWyUsTjKrhgD0dcz22mdkSnpYqbEjIm2nhwhuxlSkpywJmBo8h0ZqJdkp73mb90SssHkN4rsRaBAfAA==", - "dev": true - }, "mv": { "version": "2.1.1", "resolved": "https://registry.npmjs.org/mv/-/mv-2.1.1.tgz", @@ -4069,12 +3778,6 @@ "resolved": "https://registry.npmjs.org/negotiator/-/negotiator-0.6.2.tgz", "integrity": "sha512-hZXc7K2e+PgeI1eDBe/10Ard4ekbfrrqG8Ep+8Jmf4JID2bNg7NvCPOZN+kfF574pFQI7mum2AUqDidoKqcTOw==" }, - "nice-try": { - "version": "1.0.5", - "resolved": "https://registry.npmjs.org/nice-try/-/nice-try-1.0.5.tgz", - "integrity": "sha512-1nh45deeb5olNY7eX82BkPO7SSxR5SSYJiPTrTdFUVYwAl8CKMA5N9PjTYkHiRjisVcxcQ1HXdLhx2qxxJzLNQ==", - "dev": true - }, "nise": { "version": "4.1.0", "resolved": "https://registry.npmjs.org/nise/-/nise-4.1.0.tgz", @@ -4258,9 +3961,9 @@ "integrity": "sha1-IQmtx5ZYh8/AXLvUQsrIv7s2CGM=" }, "object-inspect": { - "version": "1.8.0", - "resolved": "https://registry.npmjs.org/object-inspect/-/object-inspect-1.8.0.tgz", - "integrity": "sha512-jLdtEOB112fORuypAyl/50VRVIBIdVQOSUUGQHzJ4xBSbit81zRarz7GThkEFZy1RceYrWYcPcBFPQwHyAc1gA==", + "version": "1.11.0", + "resolved": "https://registry.npmjs.org/object-inspect/-/object-inspect-1.11.0.tgz", + "integrity": "sha512-jp7ikS6Sd3GxQfZJPyH3cjcbJF6GZPClgdV+EFygjFLQ5FmW/dRUnTd9PQ9k0JhoNDabWFbpF1yCdSWCC6gexg==", "dev": true }, "object-keys": { @@ -4270,27 +3973,26 @@ "dev": true }, "object.assign": { - "version": "4.1.0", - "resolved": "https://registry.npmjs.org/object.assign/-/object.assign-4.1.0.tgz", - "integrity": "sha512-exHJeq6kBKj58mqGyTQ9DFvrZC/eR6OwxzoM9YRoGBqrXYonaFyGiFMuc9VZrXf7DarreEwMpurG3dd+CNyW5w==", + "version": "4.1.2", + "resolved": "https://registry.npmjs.org/object.assign/-/object.assign-4.1.2.tgz", + "integrity": "sha512-ixT2L5THXsApyiUPYKmW+2EHpXXe5Ii3M+f4e+aJFAHao5amFRW6J0OO6c/LU8Be47utCx2GL89hxGB6XSmKuQ==", "dev": true, "requires": { - "define-properties": "^1.1.2", - "function-bind": "^1.1.1", - "has-symbols": "^1.0.0", - "object-keys": "^1.0.11" + "call-bind": "^1.0.0", + "define-properties": "^1.1.3", + "has-symbols": "^1.0.1", + "object-keys": "^1.1.1" } }, "object.values": { - "version": "1.1.1", - "resolved": "https://registry.npmjs.org/object.values/-/object.values-1.1.1.tgz", - "integrity": "sha512-WTa54g2K8iu0kmS/us18jEmdv1a4Wi//BZ/DTVYEcH0XhLM5NYdpDHja3gt57VrZLcNAO2WGA+KpWsDBaHt6eA==", + "version": "1.1.4", + "resolved": "https://registry.npmjs.org/object.values/-/object.values-1.1.4.tgz", + "integrity": "sha512-TnGo7j4XSnKQoK3MfvkzqKCi0nVe/D9I9IjwTNYdb/fxYHpjrluHVOgw0AF6jrRFGMPHdfuidR09tIDiIvnaSg==", "dev": true, "requires": { + "call-bind": "^1.0.2", "define-properties": "^1.1.3", - "es-abstract": "^1.17.0-next.1", - "function-bind": "^1.1.1", - "has": "^1.0.3" + "es-abstract": "^1.18.2" } }, "on-finished": { @@ -4314,27 +4016,18 @@ "wrappy": "1" } }, - "onetime": { - "version": "5.1.0", - "resolved": "https://registry.npmjs.org/onetime/-/onetime-5.1.0.tgz", - "integrity": "sha512-5NcSkPHhwTVFIQN+TUqXoS5+dlElHXdpAWu9I0HP20YOtIi+aZ0Ct82jdlILDxjLEAWwvm+qj1m6aEtsDVmm6Q==", - "dev": true, - "requires": { - "mimic-fn": "^2.1.0" - } - }, "optionator": { - "version": "0.8.3", - "resolved": "https://registry.npmjs.org/optionator/-/optionator-0.8.3.tgz", - "integrity": "sha512-+IW9pACdk3XWmmTXG8m3upGUJst5XRGzxMRjXzAuJ1XnIFNvfhjjIuYkDvysnPQ7qzqVzLt78BCruntqRhWQbA==", + "version": "0.9.1", + "resolved": "https://registry.npmjs.org/optionator/-/optionator-0.9.1.tgz", + "integrity": "sha512-74RlY5FCnhq4jRxVUPKDaRwrVNXMqsGsiW6AJw4XK8hmtm10wC0ypZBLw5IIp85NZMr91+qd1RvvENwg7jjRFw==", "dev": true, "requires": { - "deep-is": "~0.1.3", - "fast-levenshtein": "~2.0.6", - "levn": "~0.3.0", - "prelude-ls": "~1.1.2", - "type-check": "~0.3.2", - "word-wrap": "~1.2.3" + "deep-is": "^0.1.3", + "fast-levenshtein": "^2.0.6", + "levn": "^0.4.1", + "prelude-ls": "^1.2.1", + "type-check": "^0.4.0", + "word-wrap": "^1.2.3" } }, "options": { @@ -4362,18 +4055,17 @@ } }, "p-limit": { - "version": "2.2.2", - "resolved": "https://registry.npmjs.org/p-limit/-/p-limit-2.2.2.tgz", - "integrity": "sha512-WGR+xHecKTr7EbUEhyLSh5Dube9JtdiG78ufaeLxTgpudf/20KqyMioIUZJAezlTIi6evxuoUs9YXc11cU+yzQ==", - "dev": true, + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/p-limit/-/p-limit-3.1.0.tgz", + "integrity": "sha512-TYOanM3wGwNGsZN2cVTYPArw454xnXj5qmWF1bEoAc4+cU/ol7GVh7odevjp1FNHduHc3KZMcFduxU5Xc6uJRQ==", "requires": { - "p-try": "^2.0.0" + "yocto-queue": "^0.1.0" } }, "p-locate": { "version": "2.0.0", "resolved": "https://registry.npmjs.org/p-locate/-/p-locate-2.0.0.tgz", - "integrity": "sha512-nQja7m7gSKuewoVRen45CtVfODR3crN3goVQ0DDZ9N3yHxgpkuBhZqsaiotSQRrADUrne346peY7kT3TSACykg==", + "integrity": "sha1-IKAQOyIqcMj9OcwuWAaA893l7EM=", "dev": true, "requires": { "p-limit": "^1.1.0" @@ -4387,19 +4079,13 @@ "requires": { "p-try": "^1.0.0" } - }, - "p-try": { - "version": "1.0.0", - "resolved": "https://registry.npmjs.org/p-try/-/p-try-1.0.0.tgz", - "integrity": "sha512-U1etNYuMJoIz3ZXSrrySFjsXQTWOx2/jdi86L+2pRvph/qMKL6sbcCYdH23fqsbm8TH2Gn0OybpT4eSFlCVHww==", - "dev": true } } }, "p-try": { - "version": "2.2.0", - "resolved": "https://registry.npmjs.org/p-try/-/p-try-2.2.0.tgz", - "integrity": "sha512-R4nPAVTAU0B9D35/Gk3uJf/7XYbQcyohSKdvAxIRSNghFl4e71hVoGnBNQz9cWaXxO2I10KTC+3jMdvvoKw6dQ==", + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/p-try/-/p-try-1.0.0.tgz", + "integrity": "sha1-y8ec26+P1CKOE/Yh8rGiN8GyB7M=", "dev": true }, "parent-module": { @@ -4417,12 +4103,13 @@ "integrity": "sha512-X4kUkCTHU1N/kEbwK9FpUJ0UZQa90VzeczfS704frR30gljxDG0pSziws06XlK+CGRSo/1wtG1mFIdBFQTMQNw==" }, "parse-json": { - "version": "2.2.0", - "resolved": "https://registry.npmjs.org/parse-json/-/parse-json-2.2.0.tgz", - "integrity": "sha512-QR/GGaKCkhwk1ePQNYDRKYZ3mwU9ypsKhB0XyFnLQdomyEqk3e8wpW3V5Jp88zbxK4n5ST1nqo+g9juTpownhQ==", + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/parse-json/-/parse-json-4.0.0.tgz", + "integrity": "sha1-vjX1Qlvh9/bHRxhPmKeIy5lHfuA=", "dev": true, "requires": { - "error-ex": "^1.2.0" + "error-ex": "^1.3.1", + "json-parse-better-errors": "^1.0.1" } }, "parse-ms": { @@ -4446,16 +4133,10 @@ "resolved": "https://registry.npmjs.org/path-is-absolute/-/path-is-absolute-1.0.1.tgz", "integrity": "sha512-AVbw3UJ2e9bq64vSaS9Am0fje1Pa8pbGqTTsmXfaIiMpnr5DlDhfJOuLj9Sf95ZPVDAUerDfEk88MPmPe7UCQg==" }, - "path-is-inside": { - "version": "1.0.2", - "resolved": "https://registry.npmjs.org/path-is-inside/-/path-is-inside-1.0.2.tgz", - "integrity": "sha512-DUWJr3+ULp4zXmol/SZkFf3JGsS9/SIv+Y3Rt93/UjPpDpklB5f1er4O3POIbUuUJ3FXgqte2Q7SrU6zAqwk8w==", - "dev": true - }, "path-key": { - "version": "2.0.1", - "resolved": "https://registry.npmjs.org/path-key/-/path-key-2.0.1.tgz", - "integrity": "sha512-fEHGKCSmUSDPv4uoj8AlD+joPlq3peND+HRYyxFz4KPw4z926S/b8rIuFs2FYJg3BwsxJf6A9/3eIdLaYC+9Dw==", + "version": "3.1.1", + "resolved": "https://registry.npmjs.org/path-key/-/path-key-3.1.1.tgz", + "integrity": "sha512-ojmeN0qd+y0jszEtoY48r0Peq5dwMEkIlCOu6Q5f41lfkswXuKtYrhgoTpLnyIcHm24Uhqx+5Tqm2InSwLhE6Q==", "dev": true }, "path-parse": { @@ -4469,22 +4150,28 @@ "integrity": "sha512-5DFkuoqlv1uYQKxy8omFBeJPQcdoE07Kv2sferDCrAq1ohOU+MSDswDIbnx3YAM60qIOnYa53wBhXW0EbMonrQ==" }, "path-type": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/path-type/-/path-type-2.0.0.tgz", - "integrity": "sha512-dUnb5dXUf+kzhC/W/F4e5/SkluXIFf5VUHolW1Eg1irn1hGWjPGdsRcvYJ1nD6lhk8Ir7VM0bHJKsYTx8Jx9OQ==", + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/path-type/-/path-type-3.0.0.tgz", + "integrity": "sha512-T2ZUsdZFHgA3u4e5PfPbjd7HDDpxPnQb5jN0SrDsjNSuVXHJqtwTnWqG0B1jZrgmJ/7lj1EmVIByWt1gxGkWvg==", "dev": true, "requires": { - "pify": "^2.0.0" + "pify": "^3.0.0" }, "dependencies": { "pify": { - "version": "2.3.0", - "resolved": "https://registry.npmjs.org/pify/-/pify-2.3.0.tgz", - "integrity": "sha512-udgsAY+fTnvv7kI7aaxbqwWNb0AHiB0qBO89PZKPkoTmGOgdbrHDKD+0B2X4uTfJ/FT1R09r9gTsjUjNJotuog==", + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/pify/-/pify-3.0.0.tgz", + "integrity": "sha1-5aSs0sEB/fPZpNB/DbxNtJ3SgXY=", "dev": true } } }, + "pathval": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/pathval/-/pathval-1.1.1.tgz", + "integrity": "sha512-Dp6zGqpTdETdR63lehJYPeIOqpiNBNtc7BpWSLrOje7UaIsE5aY92r/AunQA7rsXvet3lrJ3JnZX29UPTKXyKQ==", + "dev": true + }, "performance-now": { "version": "2.1.0", "resolved": "https://registry.npmjs.org/performance-now/-/performance-now-2.1.0.tgz", @@ -4504,7 +4191,16 @@ "pkg-dir": { "version": "2.0.0", "resolved": "https://registry.npmjs.org/pkg-dir/-/pkg-dir-2.0.0.tgz", - "integrity": "sha512-ojakdnUgL5pzJYWw2AIDEupaQCX5OPbM688ZevubICjdIX01PRSYKqm33fJoCOJBRseYCTUlQRnBNX+Pchaejw==", + "integrity": "sha1-9tXREJ4Z1j7fQo4L1X4Sd3YVM0s=", + "dev": true, + "requires": { + "find-up": "^2.1.0" + } + }, + "pkg-up": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/pkg-up/-/pkg-up-2.0.0.tgz", + "integrity": "sha1-yBmscoBZpGHKscOImivjxJoATX8=", "dev": true, "requires": { "find-up": "^2.1.0" @@ -4537,14 +4233,6 @@ "resolved": "https://registry.npmjs.org/delay/-/delay-4.4.1.tgz", "integrity": "sha512-aL3AhqtfhOlT/3ai6sWXeqwnw63ATNpnUiN4HL7x9q+My5QtHlO3OIkasmug9LKzpheLdmUKGRKnYXYAS7FQkQ==" }, - "p-limit": { - "version": "3.1.0", - "resolved": "https://registry.npmjs.org/p-limit/-/p-limit-3.1.0.tgz", - "integrity": "sha512-TYOanM3wGwNGsZN2cVTYPArw454xnXj5qmWF1bEoAc4+cU/ol7GVh7odevjp1FNHduHc3KZMcFduxU5Xc6uJRQ==", - "requires": { - "yocto-queue": "^0.1.0" - } - }, "protobufjs": { "version": "6.10.2", "resolved": "https://registry.npmjs.org/protobufjs/-/protobufjs-6.10.2.tgz", @@ -4573,599 +4261,17 @@ } }, "prelude-ls": { - "version": "1.1.2", - "resolved": "https://registry.npmjs.org/prelude-ls/-/prelude-ls-1.1.2.tgz", - "integrity": "sha512-ESF23V4SKG6lVSGZgYNpbsiaAkdab6ZgOxe52p7+Kid3W3u3bxR4Vfd/o21dmN7jSt0IwgZ4v5MUd26FEtXE9w==", + "version": "1.2.1", + "resolved": "https://registry.npmjs.org/prelude-ls/-/prelude-ls-1.2.1.tgz", + "integrity": "sha512-vkcDPrRZo1QZLbn5RLGPpg/WmIQ65qoWWhcGKf/b5eplkkarX0m9z8ppCat4mlOqUsWpyNuYgO3VRyrYHSzX5g==", "dev": true }, "prettier": { - "version": "2.0.5", - "resolved": "https://registry.npmjs.org/prettier/-/prettier-2.0.5.tgz", - "integrity": "sha512-7PtVymN48hGcO4fGjybyBSIWDsLU4H4XlvOHfq91pz9kkGlonzwTfYkaIEwiRg/dAJF9YlbsduBAgtYLi+8cFg==", + "version": "2.3.2", + "resolved": "https://registry.npmjs.org/prettier/-/prettier-2.3.2.tgz", + "integrity": "sha512-lnJzDfJ66zkMy58OL5/NY5zp70S7Nz6KqcKkXYzn2tMVrNxvbqaBpg7H3qHaLxCJ5lNMsGuM8+ohS7cZrthdLQ==", "dev": true }, - "prettier-eslint": { - "version": "9.0.2", - "resolved": "https://registry.npmjs.org/prettier-eslint/-/prettier-eslint-9.0.2.tgz", - "integrity": "sha512-u6EQqxUhaGfra9gy9shcR7MT7r/2twwEfRGy1tfzyaJvLQwSg34M9IU5HuF7FsLW2QUgr5VIUc56EPWibw1pdw==", - "dev": true, - "requires": { - "@typescript-eslint/parser": "^1.10.2", - "common-tags": "^1.4.0", - "core-js": "^3.1.4", - "dlv": "^1.1.0", - "eslint": "^5.0.0", - "indent-string": "^4.0.0", - "lodash.merge": "^4.6.0", - "loglevel-colored-level-prefix": "^1.0.0", - "prettier": "^1.7.0", - "pretty-format": "^23.0.1", - "require-relative": "^0.8.7", - "typescript": "^3.2.1", - "vue-eslint-parser": "^2.0.2" - }, - "dependencies": { - "ansi-escapes": { - "version": "3.2.0", - "resolved": "https://registry.npmjs.org/ansi-escapes/-/ansi-escapes-3.2.0.tgz", - "integrity": "sha512-cBhpre4ma+U0T1oM5fXg7Dy1Jw7zzwv7lt/GoCpr+hDQJoYnKVPLL4dCvSEFMmQurOQvSrwT7SL/DAlhBI97RQ==", - "dev": true - }, - "ansi-regex": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-3.0.0.tgz", - "integrity": "sha512-wFUFA5bg5dviipbQQ32yOQhl6gcJaJXiHE7dvR8VYPG97+J/GNC5FKGepKdEDUFeXRzDxPF1X/Btc8L+v7oqIQ==", - "dev": true - }, - "cli-cursor": { - "version": "2.1.0", - "resolved": "https://registry.npmjs.org/cli-cursor/-/cli-cursor-2.1.0.tgz", - "integrity": "sha512-8lgKz8LmCRYZZQDpRyT2m5rKJ08TnU4tR9FFFW2rxpxR1FzWi4PQ/NfyODchAatHaUgnSPVcx/R5w6NuTBzFiw==", - "dev": true, - "requires": { - "restore-cursor": "^2.0.0" - } - }, - "debug": { - "version": "4.1.1", - "resolved": "https://registry.npmjs.org/debug/-/debug-4.1.1.tgz", - "integrity": "sha512-pYAIzeRo8J6KPEaJ0VWOh5Pzkbw/RetuzehGM7QRRX5he4fPHx2rdKMB256ehJCkX+XRQm16eZLqLNS8RSZXZw==", - "dev": true, - "requires": { - "ms": "^2.1.1" - } - }, - "eslint": { - "version": "5.16.0", - "resolved": "https://registry.npmjs.org/eslint/-/eslint-5.16.0.tgz", - "integrity": "sha512-S3Rz11i7c8AA5JPv7xAH+dOyq/Cu/VXHiHXBPOU1k/JAM5dXqQPt3qcrhpHSorXmrpu2g0gkIBVXAqCpzfoZIg==", - "dev": true, - "requires": { - "@babel/code-frame": "^7.0.0", - "ajv": "^6.9.1", - "chalk": "^2.1.0", - "cross-spawn": "^6.0.5", - "debug": "^4.0.1", - "doctrine": "^3.0.0", - "eslint-scope": "^4.0.3", - "eslint-utils": "^1.3.1", - "eslint-visitor-keys": "^1.0.0", - "espree": "^5.0.1", - "esquery": "^1.0.1", - "esutils": "^2.0.2", - "file-entry-cache": "^5.0.1", - "functional-red-black-tree": "^1.0.1", - "glob": "^7.1.2", - "globals": "^11.7.0", - "ignore": "^4.0.6", - "import-fresh": "^3.0.0", - "imurmurhash": "^0.1.4", - "inquirer": "^6.2.2", - "js-yaml": "^3.13.0", - "json-stable-stringify-without-jsonify": "^1.0.1", - "levn": "^0.3.0", - "lodash": "^4.17.11", - "minimatch": "^3.0.4", - "mkdirp": "^0.5.1", - "natural-compare": "^1.4.0", - "optionator": "^0.8.2", - "path-is-inside": "^1.0.2", - "progress": "^2.0.0", - "regexpp": "^2.0.1", - "semver": "^5.5.1", - "strip-ansi": "^4.0.0", - "strip-json-comments": "^2.0.1", - "table": "^5.2.3", - "text-table": "^0.2.0" - } - }, - "eslint-scope": { - "version": "4.0.3", - "resolved": "https://registry.npmjs.org/eslint-scope/-/eslint-scope-4.0.3.tgz", - "integrity": "sha512-p7VutNr1O/QrxysMo3E45FjYDTeXBy0iTltPFNSqKAIfjDSXC+4dj+qfyuD8bfAXrW/y6lW3O76VaYNPKfpKrg==", - "dev": true, - "requires": { - "esrecurse": "^4.1.0", - "estraverse": "^4.1.1" - } - }, - "espree": { - "version": "5.0.1", - "resolved": "https://registry.npmjs.org/espree/-/espree-5.0.1.tgz", - "integrity": "sha512-qWAZcWh4XE/RwzLJejfcofscgMc9CamR6Tn1+XRXNzrvUSSbiAjGOI/fggztjIi7y9VLPqnICMIPiGyr8JaZ0A==", - "dev": true, - "requires": { - "acorn": "^6.0.7", - "acorn-jsx": "^5.0.0", - "eslint-visitor-keys": "^1.0.0" - } - }, - "figures": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/figures/-/figures-2.0.0.tgz", - "integrity": "sha512-Oa2M9atig69ZkfwiApY8F2Yy+tzMbazyvqv21R0NsSC8floSOC09BbT1ITWAdoMGQvJ/aZnR1KMwdx9tvHnTNA==", - "dev": true, - "requires": { - "escape-string-regexp": "^1.0.5" - } - }, - "glob": { - "version": "7.1.6", - "resolved": "https://registry.npmjs.org/glob/-/glob-7.1.6.tgz", - "integrity": "sha512-LwaxwyZ72Lk7vZINtNNrywX0ZuLyStrdDtabefZKAY5ZGJhVtgdznluResxNmPitE0SAO+O26sWTHeKSI2wMBA==", - "dev": true, - "requires": { - "fs.realpath": "^1.0.0", - "inflight": "^1.0.4", - "inherits": "2", - "minimatch": "^3.0.4", - "once": "^1.3.0", - "path-is-absolute": "^1.0.0" - } - }, - "globals": { - "version": "11.12.0", - "resolved": "https://registry.npmjs.org/globals/-/globals-11.12.0.tgz", - "integrity": "sha512-WOBp/EEGUiIsJSp7wcv/y6MO+lV9UoncWqxuFfm8eBwzWNgyfBd6Gz+IeKQ9jCmyhoH99g15M3T+QaVHFjizVA==", - "dev": true - }, - "inquirer": { - "version": "6.5.2", - "resolved": "https://registry.npmjs.org/inquirer/-/inquirer-6.5.2.tgz", - "integrity": "sha512-cntlB5ghuB0iuO65Ovoi8ogLHiWGs/5yNrtUcKjFhSSiVeAIVpD7koaSU9RM8mpXw5YDi9RdYXGQMaOURB7ycQ==", - "dev": true, - "requires": { - "ansi-escapes": "^3.2.0", - "chalk": "^2.4.2", - "cli-cursor": "^2.1.0", - "cli-width": "^2.0.0", - "external-editor": "^3.0.3", - "figures": "^2.0.0", - "lodash": "^4.17.12", - "mute-stream": "0.0.7", - "run-async": "^2.2.0", - "rxjs": "^6.4.0", - "string-width": "^2.1.0", - "strip-ansi": "^5.1.0", - "through": "^2.3.6" - }, - "dependencies": { - "ansi-regex": { - "version": "4.1.0", - "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-4.1.0.tgz", - "integrity": "sha512-1apePfXM1UOSqw0o9IiFAovVz9M5S1Dg+4TrDwfMewQ6p/rmMueb7tWZjQ1rx4Loy1ArBggoqGpfqqdI4rondg==", - "dev": true - }, - "strip-ansi": { - "version": "5.2.0", - "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-5.2.0.tgz", - "integrity": "sha512-DuRs1gKbBqsMKIZlrffwlug8MHkcnpjs5VPmL1PAh+mA30U0DTotfDZ0d2UUsXpPmPmMMJ6W773MaA3J+lbiWA==", - "dev": true, - "requires": { - "ansi-regex": "^4.1.0" - } - } - } - }, - "is-fullwidth-code-point": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/is-fullwidth-code-point/-/is-fullwidth-code-point-2.0.0.tgz", - "integrity": "sha512-VHskAKYM8RfSFXwee5t5cbN5PZeq1Wrh6qd5bkyiXIf6UQcN6w/A0eXM9r6t8d+GYOh+o6ZhiEnb88LN/Y8m2w==", - "dev": true - }, - "mimic-fn": { - "version": "1.2.0", - "resolved": "https://registry.npmjs.org/mimic-fn/-/mimic-fn-1.2.0.tgz", - "integrity": "sha512-jf84uxzwiuiIVKiOLpfYk7N46TSy8ubTonmneY9vrpHNAnp0QBt2BxWV9dO3/j+BoVAb+a5G6YDPW3M5HOdMWQ==", - "dev": true - }, - "ms": { - "version": "2.1.2", - "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.2.tgz", - "integrity": "sha512-sGkPx+VjMtmA6MX27oA4FBFELFCZZ4S4XqeGOXCv68tT+jb3vk/RyaKWP0PTKyWtmLSM0b+adUTEvbs1PEaH2w==", - "dev": true - }, - "mute-stream": { - "version": "0.0.7", - "resolved": "https://registry.npmjs.org/mute-stream/-/mute-stream-0.0.7.tgz", - "integrity": "sha512-r65nCZhrbXXb6dXOACihYApHw2Q6pV0M3V0PSxd74N0+D8nzAdEAITq2oAjA1jVnKI+tGvEBUpqiMh0+rW6zDQ==", - "dev": true - }, - "onetime": { - "version": "2.0.1", - "resolved": "https://registry.npmjs.org/onetime/-/onetime-2.0.1.tgz", - "integrity": "sha512-oyyPpiMaKARvvcgip+JV+7zci5L8D1W9RZIz2l1o08AM3pfspitVWnPt3mzHcBPp12oYMTy0pqrFs/C+m3EwsQ==", - "dev": true, - "requires": { - "mimic-fn": "^1.0.0" - } - }, - "prettier": { - "version": "1.19.1", - "resolved": "https://registry.npmjs.org/prettier/-/prettier-1.19.1.tgz", - "integrity": "sha512-s7PoyDv/II1ObgQunCbB9PdLmUcBZcnWOcxDh7O0N/UwDEsHyqkW+Qh28jW+mVuCdx7gLB0BotYI1Y6uI9iyew==", - "dev": true - }, - "restore-cursor": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/restore-cursor/-/restore-cursor-2.0.0.tgz", - "integrity": "sha512-6IzJLuGi4+R14vwagDHX+JrXmPVtPpn4mffDJ1UdR7/Edm87fl6yi8mMBIVvFtJaNTUvjughmW4hwLhRG7gC1Q==", - "dev": true, - "requires": { - "onetime": "^2.0.0", - "signal-exit": "^3.0.2" - } - }, - "semver": { - "version": "5.7.1", - "resolved": "https://registry.npmjs.org/semver/-/semver-5.7.1.tgz", - "integrity": "sha512-sauaDf/PZdVgrLTNYHRtpXa1iRiKcaebiKQ1BJdpQlWH2lCvexQdX55snPFyK7QzpudqbCI0qXFfOasHdyNDGQ==", - "dev": true - }, - "string-width": { - "version": "2.1.1", - "resolved": "https://registry.npmjs.org/string-width/-/string-width-2.1.1.tgz", - "integrity": "sha512-nOqH59deCq9SRHlxq1Aw85Jnt4w6KvLKqWVik6oA9ZklXLNIOlqg4F2yrT1MVaTjAqvVwdfeZ7w7aCvJD7ugkw==", - "dev": true, - "requires": { - "is-fullwidth-code-point": "^2.0.0", - "strip-ansi": "^4.0.0" - } - }, - "strip-ansi": { - "version": "4.0.0", - "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-4.0.0.tgz", - "integrity": "sha512-4XaJ2zQdCzROZDivEVIDPkcQn8LMFSa8kj8Gxb/Lnwzv9A8VctNZ+lfivC/sV3ivW8ElJTERXZoPBRrZKkNKow==", - "dev": true, - "requires": { - "ansi-regex": "^3.0.0" - } - }, - "strip-json-comments": { - "version": "2.0.1", - "resolved": "https://registry.npmjs.org/strip-json-comments/-/strip-json-comments-2.0.1.tgz", - "integrity": "sha512-4gB8na07fecVVkOI6Rs4e7T6NOTki5EmL7TUduTs6bu3EdnSycntVJ4re8kgZA+wx9IueI2Y11bfbgwtzuE0KQ==", - "dev": true - } - } - }, - "prettier-eslint-cli": { - "version": "5.0.0", - "resolved": "https://registry.npmjs.org/prettier-eslint-cli/-/prettier-eslint-cli-5.0.0.tgz", - "integrity": "sha512-cei9UbN1aTrz3sQs88CWpvY/10PYTevzd76zoG1tdJ164OhmNTFRKPTOZrutVvscoQWzbnLKkviS3gu5JXwvZg==", - "dev": true, - "requires": { - "arrify": "^2.0.1", - "boolify": "^1.0.0", - "camelcase-keys": "^6.0.0", - "chalk": "^2.4.2", - "common-tags": "^1.8.0", - "core-js": "^3.1.4", - "eslint": "^5.0.0", - "find-up": "^4.1.0", - "get-stdin": "^7.0.0", - "glob": "^7.1.4", - "ignore": "^5.1.2", - "lodash.memoize": "^4.1.2", - "loglevel-colored-level-prefix": "^1.0.0", - "messageformat": "^2.2.1", - "prettier-eslint": "^9.0.0", - "rxjs": "^6.5.2", - "yargs": "^13.2.4" - }, - "dependencies": { - "ansi-escapes": { - "version": "3.2.0", - "resolved": "https://registry.npmjs.org/ansi-escapes/-/ansi-escapes-3.2.0.tgz", - "integrity": "sha512-cBhpre4ma+U0T1oM5fXg7Dy1Jw7zzwv7lt/GoCpr+hDQJoYnKVPLL4dCvSEFMmQurOQvSrwT7SL/DAlhBI97RQ==", - "dev": true - }, - "ansi-regex": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-3.0.0.tgz", - "integrity": "sha512-wFUFA5bg5dviipbQQ32yOQhl6gcJaJXiHE7dvR8VYPG97+J/GNC5FKGepKdEDUFeXRzDxPF1X/Btc8L+v7oqIQ==", - "dev": true - }, - "cli-cursor": { - "version": "2.1.0", - "resolved": "https://registry.npmjs.org/cli-cursor/-/cli-cursor-2.1.0.tgz", - "integrity": "sha512-8lgKz8LmCRYZZQDpRyT2m5rKJ08TnU4tR9FFFW2rxpxR1FzWi4PQ/NfyODchAatHaUgnSPVcx/R5w6NuTBzFiw==", - "dev": true, - "requires": { - "restore-cursor": "^2.0.0" - } - }, - "debug": { - "version": "4.1.1", - "resolved": "https://registry.npmjs.org/debug/-/debug-4.1.1.tgz", - "integrity": "sha512-pYAIzeRo8J6KPEaJ0VWOh5Pzkbw/RetuzehGM7QRRX5he4fPHx2rdKMB256ehJCkX+XRQm16eZLqLNS8RSZXZw==", - "dev": true, - "requires": { - "ms": "^2.1.1" - } - }, - "eslint": { - "version": "5.16.0", - "resolved": "https://registry.npmjs.org/eslint/-/eslint-5.16.0.tgz", - "integrity": "sha512-S3Rz11i7c8AA5JPv7xAH+dOyq/Cu/VXHiHXBPOU1k/JAM5dXqQPt3qcrhpHSorXmrpu2g0gkIBVXAqCpzfoZIg==", - "dev": true, - "requires": { - "@babel/code-frame": "^7.0.0", - "ajv": "^6.9.1", - "chalk": "^2.1.0", - "cross-spawn": "^6.0.5", - "debug": "^4.0.1", - "doctrine": "^3.0.0", - "eslint-scope": "^4.0.3", - "eslint-utils": "^1.3.1", - "eslint-visitor-keys": "^1.0.0", - "espree": "^5.0.1", - "esquery": "^1.0.1", - "esutils": "^2.0.2", - "file-entry-cache": "^5.0.1", - "functional-red-black-tree": "^1.0.1", - "glob": "^7.1.2", - "globals": "^11.7.0", - "ignore": "^4.0.6", - "import-fresh": "^3.0.0", - "imurmurhash": "^0.1.4", - "inquirer": "^6.2.2", - "js-yaml": "^3.13.0", - "json-stable-stringify-without-jsonify": "^1.0.1", - "levn": "^0.3.0", - "lodash": "^4.17.11", - "minimatch": "^3.0.4", - "mkdirp": "^0.5.1", - "natural-compare": "^1.4.0", - "optionator": "^0.8.2", - "path-is-inside": "^1.0.2", - "progress": "^2.0.0", - "regexpp": "^2.0.1", - "semver": "^5.5.1", - "strip-ansi": "^4.0.0", - "strip-json-comments": "^2.0.1", - "table": "^5.2.3", - "text-table": "^0.2.0" - }, - "dependencies": { - "ignore": { - "version": "4.0.6", - "resolved": "https://registry.npmjs.org/ignore/-/ignore-4.0.6.tgz", - "integrity": "sha512-cyFDKrqc/YdcWFniJhzI42+AzS+gNwmUzOSFcRCQYwySuBBBy/KjuxWLZ/FHEH6Moq1NizMOBWyTcv8O4OZIMg==", - "dev": true - } - } - }, - "eslint-scope": { - "version": "4.0.3", - "resolved": "https://registry.npmjs.org/eslint-scope/-/eslint-scope-4.0.3.tgz", - "integrity": "sha512-p7VutNr1O/QrxysMo3E45FjYDTeXBy0iTltPFNSqKAIfjDSXC+4dj+qfyuD8bfAXrW/y6lW3O76VaYNPKfpKrg==", - "dev": true, - "requires": { - "esrecurse": "^4.1.0", - "estraverse": "^4.1.1" - } - }, - "espree": { - "version": "5.0.1", - "resolved": "https://registry.npmjs.org/espree/-/espree-5.0.1.tgz", - "integrity": "sha512-qWAZcWh4XE/RwzLJejfcofscgMc9CamR6Tn1+XRXNzrvUSSbiAjGOI/fggztjIi7y9VLPqnICMIPiGyr8JaZ0A==", - "dev": true, - "requires": { - "acorn": "^6.0.7", - "acorn-jsx": "^5.0.0", - "eslint-visitor-keys": "^1.0.0" - } - }, - "figures": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/figures/-/figures-2.0.0.tgz", - "integrity": "sha512-Oa2M9atig69ZkfwiApY8F2Yy+tzMbazyvqv21R0NsSC8floSOC09BbT1ITWAdoMGQvJ/aZnR1KMwdx9tvHnTNA==", - "dev": true, - "requires": { - "escape-string-regexp": "^1.0.5" - } - }, - "find-up": { - "version": "4.1.0", - "resolved": "https://registry.npmjs.org/find-up/-/find-up-4.1.0.tgz", - "integrity": "sha512-PpOwAdQ/YlXQ2vj8a3h8IipDuYRi3wceVQQGYWxNINccq40Anw7BlsEXCMbt1Zt+OLA6Fq9suIpIWD0OsnISlw==", - "dev": true, - "requires": { - "locate-path": "^5.0.0", - "path-exists": "^4.0.0" - } - }, - "get-stdin": { - "version": "7.0.0", - "resolved": "https://registry.npmjs.org/get-stdin/-/get-stdin-7.0.0.tgz", - "integrity": "sha512-zRKcywvrXlXsA0v0i9Io4KDRaAw7+a1ZpjRwl9Wox8PFlVCCHra7E9c4kqXCoCM9nR5tBkaTTZRBoCm60bFqTQ==", - "dev": true - }, - "glob": { - "version": "7.1.6", - "resolved": "https://registry.npmjs.org/glob/-/glob-7.1.6.tgz", - "integrity": "sha512-LwaxwyZ72Lk7vZINtNNrywX0ZuLyStrdDtabefZKAY5ZGJhVtgdznluResxNmPitE0SAO+O26sWTHeKSI2wMBA==", - "dev": true, - "requires": { - "fs.realpath": "^1.0.0", - "inflight": "^1.0.4", - "inherits": "2", - "minimatch": "^3.0.4", - "once": "^1.3.0", - "path-is-absolute": "^1.0.0" - } - }, - "globals": { - "version": "11.12.0", - "resolved": "https://registry.npmjs.org/globals/-/globals-11.12.0.tgz", - "integrity": "sha512-WOBp/EEGUiIsJSp7wcv/y6MO+lV9UoncWqxuFfm8eBwzWNgyfBd6Gz+IeKQ9jCmyhoH99g15M3T+QaVHFjizVA==", - "dev": true - }, - "ignore": { - "version": "5.1.8", - "resolved": "https://registry.npmjs.org/ignore/-/ignore-5.1.8.tgz", - "integrity": "sha512-BMpfD7PpiETpBl/A6S498BaIJ6Y/ABT93ETbby2fP00v4EbvPBXWEoaR1UBPKs3iR53pJY7EtZk5KACI57i1Uw==", - "dev": true - }, - "inquirer": { - "version": "6.5.2", - "resolved": "https://registry.npmjs.org/inquirer/-/inquirer-6.5.2.tgz", - "integrity": "sha512-cntlB5ghuB0iuO65Ovoi8ogLHiWGs/5yNrtUcKjFhSSiVeAIVpD7koaSU9RM8mpXw5YDi9RdYXGQMaOURB7ycQ==", - "dev": true, - "requires": { - "ansi-escapes": "^3.2.0", - "chalk": "^2.4.2", - "cli-cursor": "^2.1.0", - "cli-width": "^2.0.0", - "external-editor": "^3.0.3", - "figures": "^2.0.0", - "lodash": "^4.17.12", - "mute-stream": "0.0.7", - "run-async": "^2.2.0", - "rxjs": "^6.4.0", - "string-width": "^2.1.0", - "strip-ansi": "^5.1.0", - "through": "^2.3.6" - }, - "dependencies": { - "ansi-regex": { - "version": "4.1.0", - "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-4.1.0.tgz", - "integrity": "sha512-1apePfXM1UOSqw0o9IiFAovVz9M5S1Dg+4TrDwfMewQ6p/rmMueb7tWZjQ1rx4Loy1ArBggoqGpfqqdI4rondg==", - "dev": true - }, - "strip-ansi": { - "version": "5.2.0", - "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-5.2.0.tgz", - "integrity": "sha512-DuRs1gKbBqsMKIZlrffwlug8MHkcnpjs5VPmL1PAh+mA30U0DTotfDZ0d2UUsXpPmPmMMJ6W773MaA3J+lbiWA==", - "dev": true, - "requires": { - "ansi-regex": "^4.1.0" - } - } - } - }, - "is-fullwidth-code-point": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/is-fullwidth-code-point/-/is-fullwidth-code-point-2.0.0.tgz", - "integrity": "sha512-VHskAKYM8RfSFXwee5t5cbN5PZeq1Wrh6qd5bkyiXIf6UQcN6w/A0eXM9r6t8d+GYOh+o6ZhiEnb88LN/Y8m2w==", - "dev": true - }, - "locate-path": { - "version": "5.0.0", - "resolved": "https://registry.npmjs.org/locate-path/-/locate-path-5.0.0.tgz", - "integrity": "sha512-t7hw9pI+WvuwNJXwk5zVHpyhIqzg2qTlklJOf0mVxGSbe3Fp2VieZcduNYjaLDoy6p9uGpQEGWG87WpMKlNq8g==", - "dev": true, - "requires": { - "p-locate": "^4.1.0" - } - }, - "mimic-fn": { - "version": "1.2.0", - "resolved": "https://registry.npmjs.org/mimic-fn/-/mimic-fn-1.2.0.tgz", - "integrity": "sha512-jf84uxzwiuiIVKiOLpfYk7N46TSy8ubTonmneY9vrpHNAnp0QBt2BxWV9dO3/j+BoVAb+a5G6YDPW3M5HOdMWQ==", - "dev": true - }, - "ms": { - "version": "2.1.2", - "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.2.tgz", - "integrity": "sha512-sGkPx+VjMtmA6MX27oA4FBFELFCZZ4S4XqeGOXCv68tT+jb3vk/RyaKWP0PTKyWtmLSM0b+adUTEvbs1PEaH2w==", - "dev": true - }, - "mute-stream": { - "version": "0.0.7", - "resolved": "https://registry.npmjs.org/mute-stream/-/mute-stream-0.0.7.tgz", - "integrity": "sha512-r65nCZhrbXXb6dXOACihYApHw2Q6pV0M3V0PSxd74N0+D8nzAdEAITq2oAjA1jVnKI+tGvEBUpqiMh0+rW6zDQ==", - "dev": true - }, - "onetime": { - "version": "2.0.1", - "resolved": "https://registry.npmjs.org/onetime/-/onetime-2.0.1.tgz", - "integrity": "sha512-oyyPpiMaKARvvcgip+JV+7zci5L8D1W9RZIz2l1o08AM3pfspitVWnPt3mzHcBPp12oYMTy0pqrFs/C+m3EwsQ==", - "dev": true, - "requires": { - "mimic-fn": "^1.0.0" - } - }, - "p-locate": { - "version": "4.1.0", - "resolved": "https://registry.npmjs.org/p-locate/-/p-locate-4.1.0.tgz", - "integrity": "sha512-R79ZZ/0wAxKGu3oYMlz8jy/kbhsNrS7SKZ7PxEHBgJ5+F2mtFW2fK2cOtBh1cHYkQsbzFV7I+EoRKe6Yt0oK7A==", - "dev": true, - "requires": { - "p-limit": "^2.2.0" - } - }, - "path-exists": { - "version": "4.0.0", - "resolved": "https://registry.npmjs.org/path-exists/-/path-exists-4.0.0.tgz", - "integrity": "sha512-ak9Qy5Q7jYb2Wwcey5Fpvg2KoAc/ZIhLSLOSBmRmygPsGwkVVt0fZa0qrtMz+m6tJTAHfZQ8FnmB4MG4LWy7/w==", - "dev": true - }, - "restore-cursor": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/restore-cursor/-/restore-cursor-2.0.0.tgz", - "integrity": "sha512-6IzJLuGi4+R14vwagDHX+JrXmPVtPpn4mffDJ1UdR7/Edm87fl6yi8mMBIVvFtJaNTUvjughmW4hwLhRG7gC1Q==", - "dev": true, - "requires": { - "onetime": "^2.0.0", - "signal-exit": "^3.0.2" - } - }, - "semver": { - "version": "5.7.1", - "resolved": "https://registry.npmjs.org/semver/-/semver-5.7.1.tgz", - "integrity": "sha512-sauaDf/PZdVgrLTNYHRtpXa1iRiKcaebiKQ1BJdpQlWH2lCvexQdX55snPFyK7QzpudqbCI0qXFfOasHdyNDGQ==", - "dev": true - }, - "string-width": { - "version": "2.1.1", - "resolved": "https://registry.npmjs.org/string-width/-/string-width-2.1.1.tgz", - "integrity": "sha512-nOqH59deCq9SRHlxq1Aw85Jnt4w6KvLKqWVik6oA9ZklXLNIOlqg4F2yrT1MVaTjAqvVwdfeZ7w7aCvJD7ugkw==", - "dev": true, - "requires": { - "is-fullwidth-code-point": "^2.0.0", - "strip-ansi": "^4.0.0" - } - }, - "strip-ansi": { - "version": "4.0.0", - "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-4.0.0.tgz", - "integrity": "sha512-4XaJ2zQdCzROZDivEVIDPkcQn8LMFSa8kj8Gxb/Lnwzv9A8VctNZ+lfivC/sV3ivW8ElJTERXZoPBRrZKkNKow==", - "dev": true, - "requires": { - "ansi-regex": "^3.0.0" - } - }, - "strip-json-comments": { - "version": "2.0.1", - "resolved": "https://registry.npmjs.org/strip-json-comments/-/strip-json-comments-2.0.1.tgz", - "integrity": "sha512-4gB8na07fecVVkOI6Rs4e7T6NOTki5EmL7TUduTs6bu3EdnSycntVJ4re8kgZA+wx9IueI2Y11bfbgwtzuE0KQ==", - "dev": true - } - } - }, "prettier-linter-helpers": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/prettier-linter-helpers/-/prettier-linter-helpers-1.0.0.tgz", @@ -5175,24 +4281,6 @@ "fast-diff": "^1.1.2" } }, - "pretty-format": { - "version": "23.6.0", - "resolved": "https://registry.npmjs.org/pretty-format/-/pretty-format-23.6.0.tgz", - "integrity": "sha512-zf9NV1NSlDLDjycnwm6hpFATCGl/K1lt0R/GdkAK2O5LN/rwJoB+Mh93gGJjut4YbmecbfgLWVGSTCr0Ewvvbw==", - "dev": true, - "requires": { - "ansi-regex": "^3.0.0", - "ansi-styles": "^3.2.0" - }, - "dependencies": { - "ansi-regex": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-3.0.0.tgz", - "integrity": "sha512-wFUFA5bg5dviipbQQ32yOQhl6gcJaJXiHE7dvR8VYPG97+J/GNC5FKGepKdEDUFeXRzDxPF1X/Btc8L+v7oqIQ==", - "dev": true - } - } - }, "pretty-ms": { "version": "7.0.1", "resolved": "https://registry.npmjs.org/pretty-ms/-/pretty-ms-7.0.1.tgz", @@ -5313,16 +4401,10 @@ "resolved": "https://registry.npmjs.org/qs/-/qs-6.7.0.tgz", "integrity": "sha512-VCdBRNFTX1fyE7Nb6FYoURo/SPe62QCaAyzJvUjwRaIsc+NePBEniHlvxFmmX56+HZphIGtV0XeCirBtpDrTyQ==" }, - "quick-lru": { - "version": "4.0.1", - "resolved": "https://registry.npmjs.org/quick-lru/-/quick-lru-4.0.1.tgz", - "integrity": "sha512-ARhCpm70fzdcvNQfPoy49IaanKkTlRWF2JMzqhcJbhSFRZv7nPTvZJdcY7301IPmvW+/p0RgIWnQDLJxifsQ7g==", - "dev": true - }, "ramda": { - "version": "0.27.0", - "resolved": "https://registry.npmjs.org/ramda/-/ramda-0.27.0.tgz", - "integrity": "sha512-pVzZdDpWwWqEVVLshWUHjNwuVP7SfcmPraYuqocJp1yo2U1R7P+5QAfDhdItkuoGqIBnBYrtPp7rEPqDn9HlZA==", + "version": "0.27.1", + "resolved": "https://registry.npmjs.org/ramda/-/ramda-0.27.1.tgz", + "integrity": "sha512-PgIdVpn5y5Yns8vqb8FzBUEYn98V3xcPgawAkkgj0YJ0qDsnHCiNmZYfOGMgOvoB0eWFLpYbhxUR3mxfDIMvpw==", "dev": true }, "random-bytes": { @@ -5398,24 +4480,24 @@ } }, "read-pkg": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/read-pkg/-/read-pkg-2.0.0.tgz", - "integrity": "sha512-eFIBOPW7FGjzBuk3hdXEuNSiTZS/xEMlH49HxMyzb0hyPfu4EhVjT2DH32K1hSSmVq4sebAWnZuuY5auISUTGA==", + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/read-pkg/-/read-pkg-3.0.0.tgz", + "integrity": "sha1-nLxoaXj+5l0WwA4rGcI3/Pbjg4k=", "dev": true, "requires": { - "load-json-file": "^2.0.0", + "load-json-file": "^4.0.0", "normalize-package-data": "^2.3.2", - "path-type": "^2.0.0" + "path-type": "^3.0.0" } }, "read-pkg-up": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/read-pkg-up/-/read-pkg-up-2.0.0.tgz", - "integrity": "sha512-1orxQfbWGUiTn9XsPlChs6rLie/AV9jwZTGmu2NZw/CUDJQchXJFYE0Fq5j7+n558T1JhDWLdhyd1Zj+wLY//w==", + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/read-pkg-up/-/read-pkg-up-3.0.0.tgz", + "integrity": "sha1-PtSWaF26D4/hGNBpHcUfSh/5bwc=", "dev": true, "requires": { "find-up": "^2.0.0", - "read-pkg": "^2.0.0" + "read-pkg": "^3.0.0" } }, "readable-stream": { @@ -5465,9 +4547,9 @@ } }, "regexpp": { - "version": "2.0.1", - "resolved": "https://registry.npmjs.org/regexpp/-/regexpp-2.0.1.tgz", - "integrity": "sha512-lv0M6+TkDVniA3aD1Eg0DVpfU/booSu7Eev3TDO/mZKHBfVjgCGTV4t4buppESEYDtkArYFOxTJWv6S5C+iaNw==", + "version": "3.2.0", + "resolved": "https://registry.npmjs.org/regexpp/-/regexpp-3.2.0.tgz", + "integrity": "sha512-pq2bWo9mVD43nbts2wGv17XLiNLya+GklZ8kaDLV2Z08gDCsGpnKn9BFMepvWuHCbyVvY7J5o5+BVvoQbmlJLg==", "dev": true }, "request": { @@ -5533,6 +4615,12 @@ "integrity": "sha512-fGxEI7+wsG9xrvdjsrlmL22OMTTiHRwAMroiEeMgq8gzoLC/PQr7RsRDSTLUg/bZAZtF+TVIkHc6/4RIKrui+Q==", "dev": true }, + "require-from-string": { + "version": "2.0.2", + "resolved": "https://registry.npmjs.org/require-from-string/-/require-from-string-2.0.2.tgz", + "integrity": "sha512-Xf0nWe6RseziFMu+Ap9biiUbmplq6S9/p+7w7YXP/JBHhrUDDUhwa+vANyubuqfZWTveU//DYVGsDG7RKL/vEw==", + "dev": true + }, "require-in-the-middle": { "version": "5.1.0", "resolved": "https://registry.npmjs.org/require-in-the-middle/-/require-in-the-middle-5.1.0.tgz", @@ -5564,18 +4652,6 @@ "integrity": "sha512-oyrU88skkMtDdauHDuKVrgR+zuItqr6/c//FXzvmxRGMexSDc6hNvJInGW3LL46n+8b50RykrvwSUIIQH2LQ5A==", "dev": true }, - "require-main-filename": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/require-main-filename/-/require-main-filename-2.0.0.tgz", - "integrity": "sha512-NKN5kMDylKuldxYLSUfrbo5Tuzh4hd+2E8NPPX02mZtn1VuREQToYe/ZdlJy+J3uCpfaiGF05e7B8W0iXbQHmg==", - "dev": true - }, - "require-relative": { - "version": "0.8.7", - "resolved": "https://registry.npmjs.org/require-relative/-/require-relative-0.8.7.tgz", - "integrity": "sha512-AKGr4qvHiryxRb19m3PsLRGuKVAbJLUD7E6eOaHkfKhwc+vSgVOCY5xNvm9EkolBKTOf0GrQAZKLimOCz81Khg==", - "dev": true - }, "resolve": { "version": "1.15.1", "resolved": "https://registry.npmjs.org/resolve/-/resolve-1.15.1.tgz", @@ -5590,16 +4666,6 @@ "integrity": "sha512-pb/MYmXstAkysRFx8piNI1tGFNQIFA3vkE3Gq4EuA1dF6gHp/+vgZqsCGJapvy8N3Q+4o7FwvquPJcnZ7RYy4g==", "dev": true }, - "restore-cursor": { - "version": "3.1.0", - "resolved": "https://registry.npmjs.org/restore-cursor/-/restore-cursor-3.1.0.tgz", - "integrity": "sha512-l+sSefzHpj5qimhFSE5a8nufZYAM3sBSVMAPtYkmC+4EH2anSGaEMXSD0izRQbu9nfyQ9y5JrVmp7E8oZrUjvA==", - "dev": true, - "requires": { - "onetime": "^5.1.0", - "signal-exit": "^3.0.2" - } - }, "retry-request": { "version": "4.1.1", "resolved": "https://registry.npmjs.org/retry-request/-/retry-request-4.1.1.tgz", @@ -5633,21 +4699,6 @@ "glob": "^6.0.1" } }, - "run-async": { - "version": "2.4.1", - "resolved": "https://registry.npmjs.org/run-async/-/run-async-2.4.1.tgz", - "integrity": "sha512-tvVnVv01b8c1RrA6Ep7JkStj85Guv/YrMcwqYQnwjsAS2cTmmPGBBjAjpCW7RrSodNSoE2/qg9O4bceNvUuDgQ==", - "dev": true - }, - "rxjs": { - "version": "6.5.5", - "resolved": "https://registry.npmjs.org/rxjs/-/rxjs-6.5.5.tgz", - "integrity": "sha512-WfQI+1gohdf0Dai/Bbmk5L5ItH5tYqm3ki2c5GdWhKjalzjg93N3avFjVStyZZz+A2Em+ZxKH5bNghw9UeylGQ==", - "dev": true, - "requires": { - "tslib": "^1.9.0" - } - }, "safe-buffer": { "version": "5.1.1", "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.1.1.tgz", @@ -5750,18 +4801,18 @@ "integrity": "sha512-JvdAWfbXeIGaZ9cILp38HntZSFSo3mWg6xGcJJsd+d4aRMOqauag1C63dJfDw7OaMYwEbHMOxEZ1lqVRYP2OAw==" }, "shebang-command": { - "version": "1.2.0", - "resolved": "https://registry.npmjs.org/shebang-command/-/shebang-command-1.2.0.tgz", - "integrity": "sha512-EV3L1+UQWGor21OmnvojK36mhg+TyIKDh3iFBKBohr5xeXIhNBcx8oWdgkTEEQ+BEFFYdLRuqMfd5L84N1V5Vg==", + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/shebang-command/-/shebang-command-2.0.0.tgz", + "integrity": "sha512-kHxr2zZpYtdmrN1qDjrrX/Z1rR1kG8Dx+gkpK1G4eXmvXswmcE1hTWBWYUzlraYw1/yZp6YuDY77YtvbN0dmDA==", "dev": true, "requires": { - "shebang-regex": "^1.0.0" + "shebang-regex": "^3.0.0" } }, "shebang-regex": { - "version": "1.0.0", - "resolved": "https://registry.npmjs.org/shebang-regex/-/shebang-regex-1.0.0.tgz", - "integrity": "sha512-wpoSFAxys6b2a2wHZ1XpDSgD7N9iVjg29Ph9uV/uaP9Ex/KXlkTZTeddxDPSYQpgvzKLGJke2UU0AzoGCjNIvQ==", + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/shebang-regex/-/shebang-regex-3.0.0.tgz", + "integrity": "sha512-7++dFhtcx3353uBaq8DDR4NuxBetBzC7ZQOhmTQInHEd6bSrXdiEyzCvG07Z44UYdLShWUyXt5M/yhz8ekcb1A==", "dev": true }, "shimmer": { @@ -5806,20 +4857,20 @@ } }, "slice-ansi": { - "version": "2.1.0", - "resolved": "https://registry.npmjs.org/slice-ansi/-/slice-ansi-2.1.0.tgz", - "integrity": "sha512-Qu+VC3EwYLldKa1fCxuuvULvSJOKEgk9pi8dZeCVK7TqBfUNTH4sFkk4joj8afVSfAYgJoSOetjx9QWOJ5mYoQ==", + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/slice-ansi/-/slice-ansi-4.0.0.tgz", + "integrity": "sha512-qMCMfhY040cVHT43K9BFygqYbUPFZKHOg7K73mtTWJRb8pyP3fzf4Ixd5SzdEJQ6MRUg/WBnOLxghZtKKurENQ==", "dev": true, "requires": { - "ansi-styles": "^3.2.0", - "astral-regex": "^1.0.0", - "is-fullwidth-code-point": "^2.0.0" + "ansi-styles": "^4.0.0", + "astral-regex": "^2.0.0", + "is-fullwidth-code-point": "^3.0.0" }, "dependencies": { - "is-fullwidth-code-point": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/is-fullwidth-code-point/-/is-fullwidth-code-point-2.0.0.tgz", - "integrity": "sha512-VHskAKYM8RfSFXwee5t5cbN5PZeq1Wrh6qd5bkyiXIf6UQcN6w/A0eXM9r6t8d+GYOh+o6ZhiEnb88LN/Y8m2w==", + "ansi-styles": { + "version": "4.3.0", + "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-4.3.0.tgz", + "integrity": "sha512-zbB9rCJAT1rbjiVDb2hqKFHNYLxgtk8NURxZ3IZwD3F6NtxbXZQCnnSi1Lkx+IDohdPlFp222wVALIheZJQSEg==", "dev": true } } @@ -5902,9 +4953,9 @@ } }, "spdx-license-ids": { - "version": "3.0.5", - "resolved": "https://registry.npmjs.org/spdx-license-ids/-/spdx-license-ids-3.0.5.tgz", - "integrity": "sha512-J+FWzZoynJEXGphVIS+XEh3kFSjZX/1i9gFBaWQcB+/tmpe2qUsSBABpcxqxnAxFdiUFEgAX1bjYGQvIZmoz9Q==", + "version": "3.0.9", + "resolved": "https://registry.npmjs.org/spdx-license-ids/-/spdx-license-ids-3.0.9.tgz", + "integrity": "sha512-Ki212dKK4ogX+xDo4CtOZBVIwhsKBEfsEEcwmJfLQzirgc2jIWdzg40Unxz/HzEUqM1WFzVlQSMF9kZZ2HboLQ==", "dev": true }, "split": { @@ -5974,37 +5025,26 @@ "emoji-regex": "^8.0.0", "is-fullwidth-code-point": "^3.0.0", "strip-ansi": "^6.0.0" - }, - "dependencies": { - "strip-ansi": { - "version": "6.0.0", - "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-6.0.0.tgz", - "integrity": "sha512-AuvKTrTfQNYNIctbR1K/YGTR1756GycPsg7b9bdV9Duqur4gv6aKqHXah67Z8ImS7WEz5QVcOtlfW2rZEugt6w==", - "dev": true, - "requires": { - "ansi-regex": "^5.0.0" - } - } } }, "string.prototype.trimend": { - "version": "1.0.1", - "resolved": "https://registry.npmjs.org/string.prototype.trimend/-/string.prototype.trimend-1.0.1.tgz", - "integrity": "sha512-LRPxFUaTtpqYsTeNKaFOw3R4bxIzWOnbQ837QfBylo8jIxtcbK/A/sMV7Q+OAV/vWo+7s25pOE10KYSjaSO06g==", + "version": "1.0.4", + "resolved": "https://registry.npmjs.org/string.prototype.trimend/-/string.prototype.trimend-1.0.4.tgz", + "integrity": "sha512-y9xCjw1P23Awk8EvTpcyL2NIr1j7wJ39f+k6lvRnSMz+mz9CGz9NYPelDk42kOz6+ql8xjfK8oYzy3jAP5QU5A==", "dev": true, "requires": { - "define-properties": "^1.1.3", - "es-abstract": "^1.17.5" + "call-bind": "^1.0.2", + "define-properties": "^1.1.3" } }, "string.prototype.trimstart": { - "version": "1.0.1", - "resolved": "https://registry.npmjs.org/string.prototype.trimstart/-/string.prototype.trimstart-1.0.1.tgz", - "integrity": "sha512-XxZn+QpvrBI1FOcg6dIpxUPgWCPuNXvMD72aaRaUQv1eD4e/Qy8i/hFTe0BUmD60p/QA6bh1avmuPTfNjqVWRw==", + "version": "1.0.4", + "resolved": "https://registry.npmjs.org/string.prototype.trimstart/-/string.prototype.trimstart-1.0.4.tgz", + "integrity": "sha512-jh6e984OBfvxS50tdY2nRZnoC5/mLFKOREQfw8t5yytkoUsJRNxvI/E39qu1sD0OtWI3OC0XgKSmcWwziwYuZw==", "dev": true, "requires": { - "define-properties": "^1.1.3", - "es-abstract": "^1.17.5" + "call-bind": "^1.0.2", + "define-properties": "^1.1.3" } }, "string_decoder": { @@ -6016,32 +5056,24 @@ } }, "strip-ansi": { - "version": "5.2.0", - "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-5.2.0.tgz", - "integrity": "sha512-DuRs1gKbBqsMKIZlrffwlug8MHkcnpjs5VPmL1PAh+mA30U0DTotfDZ0d2UUsXpPmPmMMJ6W773MaA3J+lbiWA==", + "version": "6.0.0", + "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-6.0.0.tgz", + "integrity": "sha512-AuvKTrTfQNYNIctbR1K/YGTR1756GycPsg7b9bdV9Duqur4gv6aKqHXah67Z8ImS7WEz5QVcOtlfW2rZEugt6w==", "dev": true, "requires": { - "ansi-regex": "^4.1.0" - }, - "dependencies": { - "ansi-regex": { - "version": "4.1.0", - "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-4.1.0.tgz", - "integrity": "sha512-1apePfXM1UOSqw0o9IiFAovVz9M5S1Dg+4TrDwfMewQ6p/rmMueb7tWZjQ1rx4Loy1ArBggoqGpfqqdI4rondg==", - "dev": true - } + "ansi-regex": "^5.0.0" } }, "strip-bom": { "version": "3.0.0", "resolved": "https://registry.npmjs.org/strip-bom/-/strip-bom-3.0.0.tgz", - "integrity": "sha512-vavAMRXOgBVNF6nyEEmL3DBK19iRpDcoIwW+swQ+CbGiu7lju6t+JklA1MHweoWtadgt4ISVUsXLyDq34ddcwA==", + "integrity": "sha1-IzTBjpx1n3vdVv3vfprj1YjmjtM=", "dev": true }, "strip-json-comments": { - "version": "3.1.0", - "resolved": "https://registry.npmjs.org/strip-json-comments/-/strip-json-comments-3.1.0.tgz", - "integrity": "sha512-e6/d0eBu7gHtdCqFt0xJr642LdToM5/cN4Qb9DbHjVx1CP5RyeM+zH7pbecEmDv/lBqb0QH+6Uqq75rxFPkM0w==", + "version": "3.1.1", + "resolved": "https://registry.npmjs.org/strip-json-comments/-/strip-json-comments-3.1.1.tgz", + "integrity": "sha512-6fPc+R4ihwqP6N/aIv2f1gMH8lOVtWQHoqC4yK6oSDVVocumAsfCqjkXnqiYMhmMwS/mEHLp7Vehlt3ql6lEig==", "dev": true }, "stubs": { @@ -6059,38 +5091,28 @@ } }, "table": { - "version": "5.4.6", - "resolved": "https://registry.npmjs.org/table/-/table-5.4.6.tgz", - "integrity": "sha512-wmEc8m4fjnob4gt5riFRtTu/6+4rSe12TpAELNSqHMfF3IqnA+CH37USM6/YR3qRZv7e56kAEAtd6nKZaxe0Ug==", + "version": "6.7.1", + "resolved": "https://registry.npmjs.org/table/-/table-6.7.1.tgz", + "integrity": "sha512-ZGum47Yi6KOOFDE8m223td53ath2enHcYLgOCjGr5ngu8bdIARQk6mN/wRMv4yMRcHnCSnHbCEha4sobQx5yWg==", "dev": true, "requires": { - "ajv": "^6.10.2", - "lodash": "^4.17.14", - "slice-ansi": "^2.1.0", - "string-width": "^3.0.0" + "ajv": "^8.0.1", + "lodash.clonedeep": "^4.5.0", + "lodash.truncate": "^4.4.2", + "slice-ansi": "^4.0.0", + "string-width": "^4.2.0", + "strip-ansi": "^6.0.0" }, "dependencies": { - "emoji-regex": { - "version": "7.0.3", - "resolved": "https://registry.npmjs.org/emoji-regex/-/emoji-regex-7.0.3.tgz", - "integrity": "sha512-CwBLREIQ7LvYFB0WyRvwhq5N5qPhc6PMjD6bYggFlI5YyDgl+0vxq5VHbMOFqLg7hfWzmu8T5Z1QofhmTIhItA==", - "dev": true - }, - "is-fullwidth-code-point": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/is-fullwidth-code-point/-/is-fullwidth-code-point-2.0.0.tgz", - "integrity": "sha512-VHskAKYM8RfSFXwee5t5cbN5PZeq1Wrh6qd5bkyiXIf6UQcN6w/A0eXM9r6t8d+GYOh+o6ZhiEnb88LN/Y8m2w==", - "dev": true - }, - "string-width": { - "version": "3.1.0", - "resolved": "https://registry.npmjs.org/string-width/-/string-width-3.1.0.tgz", - "integrity": "sha512-vafcv6KjVZKSgz06oM/H6GDBrAtz8vdhQakGjFIvNrHA6y3HCF1CInLy+QLq8dTJPQ1b+KDUqDFctkdRW44e1w==", + "ajv": { + "version": "8.6.1", + "resolved": "https://registry.npmjs.org/ajv/-/ajv-8.6.1.tgz", + "integrity": "sha512-42VLtQUOLefAvKFAQIxIZDaThq6om/PrfP0CYk3/vn+y4BMNkKnbli8ON2QCiHov4KkzOSJ/xSoBJdayiiYvVQ==", "dev": true, "requires": { - "emoji-regex": "^7.0.1", - "is-fullwidth-code-point": "^2.0.0", - "strip-ansi": "^5.1.0" + "fast-deep-equal": "^3.1.1", + "require-from-string": "^2.0.2", + "uri-js": "^4.2.2" } } } @@ -6178,15 +5200,6 @@ "resolved": "https://registry.npmjs.org/tinycolor/-/tinycolor-0.0.1.tgz", "integrity": "sha1-MgtaUtg6u1l42Bo+iH1K77FaYWQ=" }, - "tmp": { - "version": "0.0.33", - "resolved": "https://registry.npmjs.org/tmp/-/tmp-0.0.33.tgz", - "integrity": "sha512-jRCJlojKnZ3addtTOjdIqoRuPEKBvNXcGYqzO6zWZX8KfKEpnGY5jfggJQ3EjKuu8D4bJRr0y+cYJFmYbImXGw==", - "dev": true, - "requires": { - "os-tmpdir": "~1.0.2" - } - }, "to-no-case": { "version": "1.0.2", "resolved": "https://registry.npmjs.org/to-no-case/-/to-no-case-1.0.2.tgz", @@ -6232,13 +5245,12 @@ } }, "tsconfig-paths": { - "version": "3.9.0", - "resolved": "https://registry.npmjs.org/tsconfig-paths/-/tsconfig-paths-3.9.0.tgz", - "integrity": "sha512-dRcuzokWhajtZWkQsDVKbWyY+jgcLC5sqJhg2PSgf4ZkH2aHPvaOY8YWGhmjb68b5qqTfasSsDO9k7RUiEmZAw==", + "version": "3.10.1", + "resolved": "https://registry.npmjs.org/tsconfig-paths/-/tsconfig-paths-3.10.1.tgz", + "integrity": "sha512-rETidPDgCpltxF7MjBZlAFPUHv5aHH2MymyPvh+vEyWAED4Eb/WeMbsnD/JDr4OKPOA1TssDHgIcpTN5Kh0p6Q==", "dev": true, "requires": { - "@types/json5": "^0.0.29", - "json5": "^1.0.1", + "json5": "^2.2.0", "minimist": "^1.2.0", "strip-bom": "^3.0.0" }, @@ -6251,12 +5263,6 @@ } } }, - "tslib": { - "version": "1.13.0", - "resolved": "https://registry.npmjs.org/tslib/-/tslib-1.13.0.tgz", - "integrity": "sha512-i/6DQjL8Xf3be4K/E6Wgpekn5Qasl1usyw++dAA35Ue5orEn65VIxOA+YvNNl9HV3qv70T7CNwjODHZrLwvd1Q==", - "dev": true - }, "tunnel-agent": { "version": "0.6.0", "resolved": "https://registry.npmjs.org/tunnel-agent/-/tunnel-agent-0.6.0.tgz", @@ -6271,24 +5277,24 @@ "integrity": "sha512-KXXFFdAbFXY4geFIwoyNK+f5Z1b7swfXABfL7HXCmoIWMKU3dmS26672A4EeQtDzLKy7SXmfBu51JolvEKwtGA==" }, "type-check": { - "version": "0.3.2", - "resolved": "https://registry.npmjs.org/type-check/-/type-check-0.3.2.tgz", - "integrity": "sha512-ZCmOJdvOWDBYJlzAoFkC+Q0+bUyEOS1ltgp1MGU03fqHG+dbi9tBFU2Rd9QKiDZFAYrhPh2JUf7rZRIuHRKtOg==", + "version": "0.4.0", + "resolved": "https://registry.npmjs.org/type-check/-/type-check-0.4.0.tgz", + "integrity": "sha512-XleUoc9uwGXqjWwXaUTZAmzMcFZ5858QA2vvx1Ur5xIcixXIP+8LnFDgRplU30us6teqdlskFfu+ae4K79Ooew==", "dev": true, "requires": { - "prelude-ls": "~1.1.2" + "prelude-ls": "^1.2.1" } }, "type-detect": { - "version": "0.1.1", - "resolved": "https://registry.npmjs.org/type-detect/-/type-detect-0.1.1.tgz", - "integrity": "sha512-5rqszGVwYgBoDkIm2oUtvkfZMQ0vk29iDMU0W2qCa3rG0vPDNczCMT4hV/bLBgLg8k8ri6+u3Zbt+S/14eMzlA==", + "version": "4.0.8", + "resolved": "https://registry.npmjs.org/type-detect/-/type-detect-4.0.8.tgz", + "integrity": "sha512-0fr/mIH1dlO+x7TlcMy+bIDqKPsw/70tVyeHW787goQjhmqaZe10uwLujubK9q9Lg6Fiho1KUKDYz0Z7k7g5/g==", "dev": true }, "type-fest": { - "version": "0.8.1", - "resolved": "https://registry.npmjs.org/type-fest/-/type-fest-0.8.1.tgz", - "integrity": "sha512-4dbzIzqvjtgiM5rw1k5rEHtBANKmdudhGyBEajN01fEyhaAIhsoKNy6y7+IN93IfpFtwY9iqi7kD+xwKhQsNJA==", + "version": "0.20.2", + "resolved": "https://registry.npmjs.org/type-fest/-/type-fest-0.20.2.tgz", + "integrity": "sha512-Ne+eE4r0/iWnpAxD852z3A+N0Bt5RN//NjJwRd2VFHEmrywxf5vsZlh4R6lixl6B+wz/8d+maTSAkN1FIkI3LQ==", "dev": true }, "type-is": { @@ -6315,12 +5321,6 @@ } } }, - "typescript": { - "version": "3.9.5", - "resolved": "https://registry.npmjs.org/typescript/-/typescript-3.9.5.tgz", - "integrity": "sha512-hSAifV3k+i6lEoCJ2k6R2Z/rp/H3+8sdmcn5NrS3/3kE7+RyZXm9aqvxWqjEXHAd8b0pShatpcdMTvEdvAJltQ==", - "dev": true - }, "uglify-js": { "version": "1.2.5", "resolved": "https://registry.npmjs.org/uglify-js/-/uglify-js-1.2.5.tgz", @@ -6334,6 +5334,18 @@ "random-bytes": "~1.0.0" } }, + "unbox-primitive": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/unbox-primitive/-/unbox-primitive-1.0.1.tgz", + "integrity": "sha512-tZU/3NqK3dA5gpE1KtyiJUrEB0lxnGkMFHptJ7q6ewdZ8s12QrODwNbhIJStmJkd1QDXa1NRA8aF2A1zk/Ypyw==", + "dev": true, + "requires": { + "function-bind": "^1.1.1", + "has-bigints": "^1.0.1", + "has-symbols": "^1.0.2", + "which-boxed-primitive": "^1.0.2" + } + }, "underscore": { "version": "1.13.1", "resolved": "https://registry.npmjs.org/underscore/-/underscore-1.13.1.tgz", @@ -6368,9 +5380,9 @@ "integrity": "sha512-DPSke0pXhTZgoF/d+WSt2QaKMCFSfx7QegxEWT+JOuHF5aWrKEn0G+ztjuJg/gG8/ItK+rbPCD/yNv8yyih6Cg==" }, "v8-compile-cache": { - "version": "2.1.1", - "resolved": "https://registry.npmjs.org/v8-compile-cache/-/v8-compile-cache-2.1.1.tgz", - "integrity": "sha512-8OQ9CL+VWyt3JStj7HX7/ciTL2V3Rl1Wf5OL+SNTm0yK1KvtReVulksyeRnCANHHuUxHlQig+JJDlUhBt1NQDQ==", + "version": "2.3.0", + "resolved": "https://registry.npmjs.org/v8-compile-cache/-/v8-compile-cache-2.3.0.tgz", + "integrity": "sha512-l8lCEmLcLYZh4nbunNZvQCJc5pv7+RCwa8q/LdUx8u7lsWvPDKmpodJAJNwkAhJC//dFY48KuIEmjtd4RViDrA==", "dev": true }, "validate-npm-package-license": { @@ -6398,99 +5410,32 @@ "extsprintf": "^1.2.0" } }, - "vue-eslint-parser": { - "version": "2.0.3", - "resolved": "https://registry.npmjs.org/vue-eslint-parser/-/vue-eslint-parser-2.0.3.tgz", - "integrity": "sha512-ZezcU71Owm84xVF6gfurBQUGg8WQ+WZGxgDEQu1IHFBZNx7BFZg3L1yHxrCBNNwbwFtE1GuvfJKMtb6Xuwc/Bw==", - "dev": true, - "requires": { - "debug": "^3.1.0", - "eslint-scope": "^3.7.1", - "eslint-visitor-keys": "^1.0.0", - "espree": "^3.5.2", - "esquery": "^1.0.0", - "lodash": "^4.17.4" - }, - "dependencies": { - "acorn": { - "version": "5.7.4", - "resolved": "https://registry.npmjs.org/acorn/-/acorn-5.7.4.tgz", - "integrity": "sha512-1D++VG7BhrtvQpNbBzovKNc1FLGGEE/oGe7b9xJm/RFHMBeUaUGpluV9RLjZa47YFdPcDAenEYuq9pQPcMdLJg==", - "dev": true - }, - "acorn-jsx": { - "version": "3.0.1", - "resolved": "https://registry.npmjs.org/acorn-jsx/-/acorn-jsx-3.0.1.tgz", - "integrity": "sha512-AU7pnZkguthwBjKgCg6998ByQNIMjbuDQZ8bb78QAFZwPfmKia8AIzgY/gWgqCjnht8JLdXmB4YxA0KaV60ncQ==", - "dev": true, - "requires": { - "acorn": "^3.0.4" - }, - "dependencies": { - "acorn": { - "version": "3.3.0", - "resolved": "https://registry.npmjs.org/acorn/-/acorn-3.3.0.tgz", - "integrity": "sha512-OLUyIIZ7mF5oaAUT1w0TFqQS81q3saT46x8t7ukpPjMNk+nbs4ZHhs7ToV8EWnLYLepjETXd4XaCE4uxkMeqUw==", - "dev": true - } - } - }, - "debug": { - "version": "3.2.6", - "resolved": "https://registry.npmjs.org/debug/-/debug-3.2.6.tgz", - "integrity": "sha512-mel+jf7nrtEl5Pn1Qx46zARXKDpBbvzezse7p7LqINmdoIk8PYP5SySaxEmYv6TZ0JyEKA1hsCId6DIhgITtWQ==", - "dev": true, - "requires": { - "ms": "^2.1.1" - } - }, - "eslint-scope": { - "version": "3.7.3", - "resolved": "https://registry.npmjs.org/eslint-scope/-/eslint-scope-3.7.3.tgz", - "integrity": "sha512-W+B0SvF4gamyCTmUc+uITPY0989iXVfKvhwtmJocTaYoc/3khEHmEmvfY/Gn9HA9VV75jrQECsHizkNw1b68FA==", - "dev": true, - "requires": { - "esrecurse": "^4.1.0", - "estraverse": "^4.1.1" - } - }, - "espree": { - "version": "3.5.4", - "resolved": "https://registry.npmjs.org/espree/-/espree-3.5.4.tgz", - "integrity": "sha512-yAcIQxtmMiB/jL32dzEp2enBeidsB7xWPLNiw3IIkpVds1P+h7qF9YwJq1yUNzp2OKXgAprs4F61ih66UsoD1A==", - "dev": true, - "requires": { - "acorn": "^5.5.0", - "acorn-jsx": "^3.0.0" - } - }, - "ms": { - "version": "2.1.2", - "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.2.tgz", - "integrity": "sha512-sGkPx+VjMtmA6MX27oA4FBFELFCZZ4S4XqeGOXCv68tT+jb3vk/RyaKWP0PTKyWtmLSM0b+adUTEvbs1PEaH2w==", - "dev": true - } - } - }, "walkdir": { "version": "0.4.1", "resolved": "https://registry.npmjs.org/walkdir/-/walkdir-0.4.1.tgz", "integrity": "sha512-3eBwRyEln6E1MSzcxcVpQIhRG8Q1jLvEqRmCZqS3dsfXEDR/AhOF4d+jHg1qvDCpYaVRZjENPQyrVxAkQqxPgQ==" }, "which": { - "version": "1.3.1", - "resolved": "https://registry.npmjs.org/which/-/which-1.3.1.tgz", - "integrity": "sha512-HxJdYWq1MTIQbJ3nw0cqssHoTNU267KlrDuGZ1WYlxDStUtKUhOaJmh112/TZmHxxUfuJqPXSOm7tDyas0OSIQ==", + "version": "2.0.2", + "resolved": "https://registry.npmjs.org/which/-/which-2.0.2.tgz", + "integrity": "sha512-BLI3Tl1TW3Pvl70l3yq3Y64i+awpwXqsGBYWkkqMtnbXgrMD+yj7rhW0kuEDxzJaYXGjEW5ogapKNMEKNMjibA==", "dev": true, "requires": { "isexe": "^2.0.0" } }, - "which-module": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/which-module/-/which-module-2.0.0.tgz", - "integrity": "sha512-B+enWhmw6cjfVC7kS8Pj9pCrKSc5txArRyaYGe088shv/FGWH+0Rjx/xPgtsWfsUtS27FkP697E4DDhgrgoc0Q==", - "dev": true + "which-boxed-primitive": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/which-boxed-primitive/-/which-boxed-primitive-1.0.2.tgz", + "integrity": "sha512-bwZdv0AKLpplFY2KZRX6TvyuN7ojjr7lwkg6ml0roIy9YeuSr7JS372qlNW18UQYzgYK9ziGcerWqZOmEn9VNg==", + "dev": true, + "requires": { + "is-bigint": "^1.0.1", + "is-boolean-object": "^1.1.0", + "is-number-object": "^1.0.4", + "is-string": "^1.0.5", + "is-symbol": "^1.0.3" + } }, "wide-align": { "version": "1.1.3", @@ -6542,39 +5487,13 @@ "dev": true }, "wrap-ansi": { - "version": "5.1.0", - "resolved": "https://registry.npmjs.org/wrap-ansi/-/wrap-ansi-5.1.0.tgz", - "integrity": "sha512-QC1/iN/2/RPVJ5jYK8BGttj5z83LmSKmvbvrXPNCLZSEb32KKVDJDl/MOt2N01qU2H/FkzEa9PKto1BqDjtd7Q==", + "version": "7.0.0", + "resolved": "https://registry.npmjs.org/wrap-ansi/-/wrap-ansi-7.0.0.tgz", + "integrity": "sha512-YVGIj2kamLSTxw6NsZjoBxfSwsn0ycdesmc4p+Q21c5zPuZ1pl+NfxVdxPtdHvmNVOQ6XSYG4AUtyt/Fi7D16Q==", "dev": true, "requires": { - "ansi-styles": "^3.2.0", - "string-width": "^3.0.0", - "strip-ansi": "^5.0.0" - }, - "dependencies": { - "emoji-regex": { - "version": "7.0.3", - "resolved": "https://registry.npmjs.org/emoji-regex/-/emoji-regex-7.0.3.tgz", - "integrity": "sha512-CwBLREIQ7LvYFB0WyRvwhq5N5qPhc6PMjD6bYggFlI5YyDgl+0vxq5VHbMOFqLg7hfWzmu8T5Z1QofhmTIhItA==", - "dev": true - }, - "is-fullwidth-code-point": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/is-fullwidth-code-point/-/is-fullwidth-code-point-2.0.0.tgz", - "integrity": "sha512-VHskAKYM8RfSFXwee5t5cbN5PZeq1Wrh6qd5bkyiXIf6UQcN6w/A0eXM9r6t8d+GYOh+o6ZhiEnb88LN/Y8m2w==", - "dev": true - }, - "string-width": { - "version": "3.1.0", - "resolved": "https://registry.npmjs.org/string-width/-/string-width-3.1.0.tgz", - "integrity": "sha512-vafcv6KjVZKSgz06oM/H6GDBrAtz8vdhQakGjFIvNrHA6y3HCF1CInLy+QLq8dTJPQ1b+KDUqDFctkdRW44e1w==", - "dev": true, - "requires": { - "emoji-regex": "^7.0.1", - "is-fullwidth-code-point": "^2.0.0", - "strip-ansi": "^5.1.0" - } - } + "string-width": "^4.1.0", + "strip-ansi": "^6.0.0" } }, "wrappy": { @@ -6582,15 +5501,6 @@ "resolved": "https://registry.npmjs.org/wrappy/-/wrappy-1.0.2.tgz", "integrity": "sha512-l4Sp/DRseor9wL6EvV2+TuQn63dMkPjZ/sp9XkghTEbV9KlPS1xUsZ3u7/IQO4wxtcFB4bgpQPRcR3QCvezPcQ==" }, - "write": { - "version": "1.0.3", - "resolved": "https://registry.npmjs.org/write/-/write-1.0.3.tgz", - "integrity": "sha512-/lg70HAjtkUgWPVZhZcm+T4hkL8Zbtp1nFNOn3lRrxnlv50SRBv7cR7RqR+GMsd3hUXy9hWBo4CHTbFTcOYwig==", - "dev": true, - "requires": { - "mkdirp": "^0.5.1" - } - }, "ws": { "version": "0.4.32", "resolved": "https://registry.npmjs.org/ws/-/ws-0.4.32.tgz", @@ -6626,91 +5536,25 @@ "integrity": "sha512-a4UGQaWPH59mOXUYnAG2ewncQS4i4F43Tv3JoAM+s2VDAmS9NsK8GpDMLrCHPksFT7h3K6TOoUNn2pb7RoXx4g==" }, "yargs": { - "version": "13.3.2", - "resolved": "https://registry.npmjs.org/yargs/-/yargs-13.3.2.tgz", - "integrity": "sha512-AX3Zw5iPruN5ie6xGRIDgqkT+ZhnRlZMLMHAs8tg7nRruy2Nb+i5o9bwghAogtM08q1dpr2LVoS8KSTMYpWXUw==", + "version": "16.2.0", + "resolved": "https://registry.npmjs.org/yargs/-/yargs-16.2.0.tgz", + "integrity": "sha512-D1mvvtDG0L5ft/jGWkLpG1+m0eQxOfaBvTNELraWj22wSVUMWxZUvYgJYcKh6jGGIkJFhH4IZPQhR4TKpc8mBw==", "dev": true, "requires": { - "cliui": "^5.0.0", - "find-up": "^3.0.0", - "get-caller-file": "^2.0.1", + "cliui": "^7.0.2", + "escalade": "^3.1.1", + "get-caller-file": "^2.0.5", "require-directory": "^2.1.1", - "require-main-filename": "^2.0.0", - "set-blocking": "^2.0.0", - "string-width": "^3.0.0", - "which-module": "^2.0.0", - "y18n": "^4.0.0", - "yargs-parser": "^13.1.2" - }, - "dependencies": { - "emoji-regex": { - "version": "7.0.3", - "resolved": "https://registry.npmjs.org/emoji-regex/-/emoji-regex-7.0.3.tgz", - "integrity": "sha512-CwBLREIQ7LvYFB0WyRvwhq5N5qPhc6PMjD6bYggFlI5YyDgl+0vxq5VHbMOFqLg7hfWzmu8T5Z1QofhmTIhItA==", - "dev": true - }, - "find-up": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/find-up/-/find-up-3.0.0.tgz", - "integrity": "sha512-1yD6RmLI1XBfxugvORwlck6f75tYL+iR0jqwsOrOxMZyGYqUuDhJ0l4AXdO1iX/FTs9cBAMEk1gWSEx1kSbylg==", - "dev": true, - "requires": { - "locate-path": "^3.0.0" - } - }, - "is-fullwidth-code-point": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/is-fullwidth-code-point/-/is-fullwidth-code-point-2.0.0.tgz", - "integrity": "sha512-VHskAKYM8RfSFXwee5t5cbN5PZeq1Wrh6qd5bkyiXIf6UQcN6w/A0eXM9r6t8d+GYOh+o6ZhiEnb88LN/Y8m2w==", - "dev": true - }, - "locate-path": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/locate-path/-/locate-path-3.0.0.tgz", - "integrity": "sha512-7AO748wWnIhNqAuaty2ZWHkQHRSNfPVIsPIfwEOWO22AmaoVrWavlOcMR5nzTLNYvp36X220/maaRsrec1G65A==", - "dev": true, - "requires": { - "p-locate": "^3.0.0", - "path-exists": "^3.0.0" - } - }, - "p-locate": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/p-locate/-/p-locate-3.0.0.tgz", - "integrity": "sha512-x+12w/To+4GFfgJhBEpiDcLozRJGegY+Ei7/z0tSLkMmxGZNybVMSfWj9aJn8Z5Fc7dBUNJOOVgPv2H7IwulSQ==", - "dev": true, - "requires": { - "p-limit": "^2.0.0" - } - }, - "string-width": { - "version": "3.1.0", - "resolved": "https://registry.npmjs.org/string-width/-/string-width-3.1.0.tgz", - "integrity": "sha512-vafcv6KjVZKSgz06oM/H6GDBrAtz8vdhQakGjFIvNrHA6y3HCF1CInLy+QLq8dTJPQ1b+KDUqDFctkdRW44e1w==", - "dev": true, - "requires": { - "emoji-regex": "^7.0.1", - "is-fullwidth-code-point": "^2.0.0", - "strip-ansi": "^5.1.0" - } - }, - "y18n": { - "version": "4.0.1", - "resolved": "https://registry.npmjs.org/y18n/-/y18n-4.0.1.tgz", - "integrity": "sha512-wNcy4NvjMYL8gogWWYAO7ZFWFfHcbdbE57tZO8e4cbpj8tfUcwrwqSl3ad8HxpYWCdXcJUCeKKZS62Av1affwQ==", - "dev": true - } + "string-width": "^4.2.0", + "y18n": "^5.0.5", + "yargs-parser": "^20.2.2" } }, "yargs-parser": { - "version": "13.1.2", - "resolved": "https://registry.npmjs.org/yargs-parser/-/yargs-parser-13.1.2.tgz", - "integrity": "sha512-3lbsNRf/j+A4QuSZfDRA7HRSfWrzO0YjqTJd5kjAq37Zep1CEgaYmrH9Q3GwPiB9cHyd1Y1UwggGhJGoxipbzg==", - "dev": true, - "requires": { - "camelcase": "^5.0.0", - "decamelize": "^1.2.0" - } + "version": "20.2.4", + "resolved": "https://registry.npmjs.org/yargs-parser/-/yargs-parser-20.2.4.tgz", + "integrity": "sha512-WOkpgNhPTlE73h4VFAFsOnomJVaovO8VqLDzy5saChRBFQFBoMYirowyW+Q9HB4HFF4Z7VZTiG3iSzJJA29yRA==", + "dev": true }, "yargs-unparser": { "version": "2.0.0", @@ -6722,20 +5566,6 @@ "decamelize": "^4.0.0", "flat": "^5.0.2", "is-plain-obj": "^2.1.0" - }, - "dependencies": { - "camelcase": { - "version": "6.2.0", - "resolved": "https://registry.npmjs.org/camelcase/-/camelcase-6.2.0.tgz", - "integrity": "sha512-c7wVvbw3f37nuobQNtgsgG9POC9qMbNuMQmTCqZv23b6MIz0fcYpBiOlv9gEN/hdLdnZTDQhg6e9Dq5M1vKvfg==", - "dev": true - }, - "decamelize": { - "version": "4.0.0", - "resolved": "https://registry.npmjs.org/decamelize/-/decamelize-4.0.0.tgz", - "integrity": "sha512-9iE1PgSik9HeIIw2JO94IidnE3eBoQrFJ3w7sFuzSX4DpmZ3v5sZpUiV5Swcf6mQEF+Y0ru8Neo+p+nyh2J+hQ==", - "dev": true - } } }, "yn": { diff --git a/services/real-time/package.json b/services/real-time/package.json index 37cd18f81f..308957330d 100644 --- a/services/real-time/package.json +++ b/services/real-time/package.json @@ -16,9 +16,10 @@ "test:unit:_run": "mocha --recursive --reporter spec $@ test/unit/js", "test:unit": "npm run test:unit:_run -- --grep=$MOCHA_GREP", "nodemon": "nodemon --config nodemon.json", - "lint": "node_modules/.bin/eslint --max-warnings 0 .", - "format": "node_modules/.bin/prettier-eslint $PWD'/**/*.js' --list-different", - "format:fix": "node_modules/.bin/prettier-eslint $PWD'/**/*.js' --write" + "lint": "eslint --max-warnings 0 --format unix .", + "format": "prettier --list-different $PWD/'**/*.js'", + "format:fix": "prettier --write $PWD/'**/*.js'", + "lint:fix": "eslint --fix ." }, "dependencies": { "@overleaf/metrics": "^3.5.1", @@ -41,22 +42,21 @@ "underscore": "1.13.1" }, "devDependencies": { - "chai": "~1.9.1", + "chai": "^4.2.0", + "chai-as-promised": "^7.1.1", "cookie-signature": "^1.1.0", - "eslint": "^6.8.0", - "eslint-config-prettier": "^6.10.0", - "eslint-config-standard": "^14.1.0", - "eslint-plugin-chai-expect": "^2.1.0", - "eslint-plugin-chai-friendly": "^0.5.0", - "eslint-plugin-import": "^2.20.1", - "eslint-plugin-mocha": "^6.3.0", - "eslint-plugin-node": "^11.0.0", + "eslint": "^7.21.0", + "eslint-config-prettier": "^8.1.0", + "eslint-config-standard": "^16.0.2", + "eslint-plugin-chai-expect": "^2.2.0", + "eslint-plugin-chai-friendly": "^0.6.0", + "eslint-plugin-import": "^2.22.1", + "eslint-plugin-mocha": "^8.0.0", + "eslint-plugin-node": "^11.1.0", "eslint-plugin-prettier": "^3.1.2", "eslint-plugin-promise": "^4.2.1", - "eslint-plugin-standard": "^4.0.1", "mocha": "^8.3.2", - "prettier": "^2.0.0", - "prettier-eslint-cli": "^5.0.0", + "prettier": "^2.2.1", "sandboxed-module": "~0.3.0", "sinon": "^9.2.4", "timekeeper": "0.0.4", From 7e8e231059a696225ffcce080ab00cbacba17c4f Mon Sep 17 00:00:00 2001 From: Jakob Ackermann Date: Tue, 13 Jul 2021 12:04:45 +0100 Subject: [PATCH 487/491] [misc] run format_fix and lint:fix --- services/real-time/app.js | 16 +- .../real-time/app/js/AuthorizationManager.js | 2 +- services/real-time/app/js/ChannelManager.js | 4 +- .../real-time/app/js/ConnectedUsersManager.js | 100 ++--- .../real-time/app/js/DeploymentManager.js | 4 +- .../app/js/DocumentUpdaterController.js | 16 +- .../app/js/DocumentUpdaterManager.js | 38 +- services/real-time/app/js/DrainManager.js | 2 +- services/real-time/app/js/Errors.js | 8 +- services/real-time/app/js/EventLogger.js | 2 +- .../real-time/app/js/HttpApiController.js | 2 +- services/real-time/app/js/HttpController.js | 10 +- .../real-time/app/js/RedisClientManager.js | 4 +- services/real-time/app/js/RoomManager.js | 8 +- services/real-time/app/js/Router.js | 93 +++-- services/real-time/app/js/SafeJsonParse.js | 2 +- services/real-time/app/js/SessionSockets.js | 2 +- services/real-time/app/js/WebApiManager.js | 8 +- .../real-time/app/js/WebsocketController.js | 384 +++++++++--------- .../real-time/app/js/WebsocketLoadBalancer.js | 25 +- .../real-time/config/settings.defaults.js | 32 +- services/real-time/config/settings.test.js | 4 +- .../test/acceptance/js/ApplyUpdateTests.js | 118 +++--- .../test/acceptance/js/ClientTrackingTests.js | 72 ++-- .../test/acceptance/js/DrainManagerTests.js | 24 +- .../test/acceptance/js/EarlyDisconnect.js | 60 +-- .../test/acceptance/js/HttpControllerTests.js | 22 +- .../test/acceptance/js/JoinDocTests.js | 108 ++--- .../test/acceptance/js/JoinProjectTests.js | 58 +-- .../test/acceptance/js/LeaveDocTests.js | 20 +- .../test/acceptance/js/LeaveProjectTests.js | 50 +-- .../test/acceptance/js/MatrixTests.js | 96 ++--- .../test/acceptance/js/PubSubRace.js | 72 ++-- .../test/acceptance/js/ReceiveUpdateTests.js | 82 ++-- .../test/acceptance/js/RouterTests.js | 34 +- .../test/acceptance/js/SessionSocketsTests.js | 12 +- .../test/acceptance/js/SessionTests.js | 4 +- .../acceptance/js/helpers/FixturesManager.js | 25 +- .../js/helpers/MockDocUpdaterServer.js | 8 +- .../acceptance/js/helpers/MockWebServer.js | 8 +- .../acceptance/js/helpers/RealTimeClient.js | 32 +- .../acceptance/js/helpers/RealtimeServer.js | 6 +- .../test/acceptance/libs/XMLHttpRequest.js | 82 ++-- services/real-time/test/setup.js | 10 +- .../test/unit/js/AuthorizationManagerTests.js | 34 +- .../test/unit/js/ChannelManagerTests.js | 8 +- .../unit/js/ConnectedUsersManagerTests.js | 58 +-- .../unit/js/DocumentUpdaterControllerTests.js | 46 ++- .../unit/js/DocumentUpdaterManagerTests.js | 34 +- .../test/unit/js/DrainManagerTests.js | 20 +- .../test/unit/js/EventLoggerTests.js | 4 +- .../test/unit/js/RoomManagerTests.js | 18 +- .../test/unit/js/SafeJsonParseTest.js | 6 +- .../test/unit/js/SessionSocketsTests.js | 24 +- .../test/unit/js/WebApiManagerTests.js | 30 +- .../test/unit/js/WebsocketControllerTests.js | 100 ++--- .../unit/js/WebsocketLoadBalancerTests.js | 60 +-- 57 files changed, 1104 insertions(+), 1107 deletions(-) diff --git a/services/real-time/app.js b/services/real-time/app.js index 0b8be4d17b..31216bb07e 100644 --- a/services/real-time/app.js +++ b/services/real-time/app.js @@ -34,7 +34,7 @@ const socketIoLogger = { }, info() {}, debug() {}, - log() {} + log() {}, } // monitor status file to take dark deployments out of the load-balancer @@ -45,7 +45,7 @@ const app = express() const server = require('http').createServer(app) const io = require('socket.io').listen(server, { - logger: socketIoLogger + logger: socketIoLogger, }) // Bind to sessions @@ -78,7 +78,7 @@ io.configure(function () { 'flashsocket', 'htmlfile', 'xhr-polling', - 'jsonp-polling' + 'jsonp-polling', ]) }) @@ -195,10 +195,10 @@ function drainAndShutdown(signal) { const staleClients = io.sockets.clients() if (staleClients.length !== 0) { logger.warn( - { staleClients: staleClients.map((client) => client.id) }, + { staleClients: staleClients.map(client => client.id) }, 'forcefully disconnecting stale clients' ) - staleClients.forEach((client) => { + staleClients.forEach(client => { client.disconnect() }) } @@ -222,7 +222,7 @@ if (Settings.shutdownDrainTimeWindow) { 'SIGUSR1', 'SIGUSR2', 'SIGTERM', - 'SIGABRT' + 'SIGABRT', ]) { process.on(signal, drainAndShutdown) } // signal is passed as argument to event handler @@ -237,7 +237,7 @@ if (Settings.shutdownDrainTimeWindow) { 'EHOSTUNREACH', 'EPIPE', 'ECONNRESET', - 'ERR_STREAM_WRITE_AFTER_END' + 'ERR_STREAM_WRITE_AFTER_END', ].includes(error.code) ) { Metrics.inc('disconnected_write', 1, { status: error.code }) @@ -266,7 +266,7 @@ if (Settings.continualPubsubTraffic) { const json = JSON.stringify({ health_check: true, key: checker.id, - date: new Date().toString() + date: new Date().toString(), }) Metrics.summary(`redis.publish.${channel}`, json.length) pubsubClient.publish(channel, json, function (err) { diff --git a/services/real-time/app/js/AuthorizationManager.js b/services/real-time/app/js/AuthorizationManager.js index 0c37ac1423..4b48ed0e8c 100644 --- a/services/real-time/app/js/AuthorizationManager.js +++ b/services/real-time/app/js/AuthorizationManager.js @@ -63,5 +63,5 @@ module.exports = AuthorizationManager = { removeAccessToDoc(client, doc_id, callback) { delete client.ol_context[`doc:${doc_id}`] callback(null) - } + }, } diff --git a/services/real-time/app/js/ChannelManager.js b/services/real-time/app/js/ChannelManager.js index c777fb2250..81caa7dd83 100644 --- a/services/real-time/app/js/ChannelManager.js +++ b/services/real-time/app/js/ChannelManager.js @@ -39,7 +39,7 @@ module.exports = { metrics.inc(`subscribe.failed.${baseChannel}`) // add context for the stack-trace at the call-site throw new OError('failed to subscribe to channel', { - channel + channel, }).withCause(err) }) } @@ -97,5 +97,5 @@ module.exports = { // we publish on a different client to the subscribe, so we can't // check for the channel existing here rclient.publish(channel, data) - } + }, } diff --git a/services/real-time/app/js/ConnectedUsersManager.js b/services/real-time/app/js/ConnectedUsersManager.js index 006e763645..f8dd40fd6a 100644 --- a/services/real-time/app/js/ConnectedUsersManager.js +++ b/services/real-time/app/js/ConnectedUsersManager.js @@ -111,66 +111,66 @@ module.exports = { }, _getConnectedUser(project_id, client_id, callback) { - rclient.hgetall(Keys.connectedUser({ project_id, client_id }), function ( - err, - result - ) { - if (err) { - err = new OError('problem fetching connected user details', { - other_client_id: client_id - }).withCause(err) - return callback(err) - } - if (!(result && result.user_id)) { - result = { - connected: false, - client_id + rclient.hgetall( + Keys.connectedUser({ project_id, client_id }), + function (err, result) { + if (err) { + err = new OError('problem fetching connected user details', { + other_client_id: client_id, + }).withCause(err) + return callback(err) } - } else { - result.connected = true - result.client_id = client_id - result.client_age = - (Date.now() - parseInt(result.last_updated_at, 10)) / 1000 - if (result.cursorData) { - try { - result.cursorData = JSON.parse(result.cursorData) - } catch (e) { - OError.tag(e, 'error parsing cursorData JSON', { - other_client_id: client_id, - cursorData: result.cursorData - }) - return callback(e) + if (!(result && result.user_id)) { + result = { + connected: false, + client_id, + } + } else { + result.connected = true + result.client_id = client_id + result.client_age = + (Date.now() - parseInt(result.last_updated_at, 10)) / 1000 + if (result.cursorData) { + try { + result.cursorData = JSON.parse(result.cursorData) + } catch (e) { + OError.tag(e, 'error parsing cursorData JSON', { + other_client_id: client_id, + cursorData: result.cursorData, + }) + return callback(e) + } } } + callback(err, result) } - callback(err, result) - }) + ) }, getConnectedUsers(project_id, callback) { const self = this - rclient.smembers(Keys.clientsInProject({ project_id }), function ( - err, - results - ) { - if (err) { - err = new OError('problem getting clients in project').withCause(err) - return callback(err) - } - const jobs = results.map((client_id) => (cb) => - self._getConnectedUser(project_id, client_id, cb) - ) - async.series(jobs, function (err, users) { + rclient.smembers( + Keys.clientsInProject({ project_id }), + function (err, results) { if (err) { - OError.tag(err, 'problem getting connected users') + err = new OError('problem getting clients in project').withCause(err) return callback(err) } - users = users.filter( - (user) => - user && user.connected && user.client_age < REFRESH_TIMEOUT_IN_S + const jobs = results.map( + client_id => cb => self._getConnectedUser(project_id, client_id, cb) ) - callback(null, users) - }) - }) - } + async.series(jobs, function (err, users) { + if (err) { + OError.tag(err, 'problem getting connected users') + return callback(err) + } + users = users.filter( + user => + user && user.connected && user.client_age < REFRESH_TIMEOUT_IN_S + ) + callback(null, users) + }) + } + ) + }, } diff --git a/services/real-time/app/js/DeploymentManager.js b/services/real-time/app/js/DeploymentManager.js index fd482740f5..0e6b2a7f05 100644 --- a/services/real-time/app/js/DeploymentManager.js +++ b/services/real-time/app/js/DeploymentManager.js @@ -9,7 +9,7 @@ const FILE_CHECK_INTERVAL = 5000 const statusFile = settings.deploymentFile const deploymentColour = settings.deploymentColour -var serviceCloseTime +let serviceCloseTime function updateDeploymentStatus(fileContent) { const closed = fileContent && !fileContent.includes(deploymentColour) @@ -55,5 +55,5 @@ module.exports = { }, deploymentIsClosed() { return settings.serviceIsClosed && Date.now() > serviceCloseTime - } + }, } diff --git a/services/real-time/app/js/DocumentUpdaterController.js b/services/real-time/app/js/DocumentUpdaterController.js index 7ce516524c..290882c41f 100644 --- a/services/real-time/app/js/DocumentUpdaterController.js +++ b/services/real-time/app/js/DocumentUpdaterController.js @@ -50,7 +50,7 @@ module.exports = DocumentUpdaterController = { handleRoomUpdates(rclientSubList) { const roomEvents = RoomManager.eventSource() roomEvents.on('doc-active', function (doc_id) { - const subscribePromises = rclientSubList.map((rclient) => + const subscribePromises = rclientSubList.map(rclient => ChannelManager.subscribe(rclient, 'applied-ops', doc_id) ) RoomManager.emitOnCompletion( @@ -58,8 +58,8 @@ module.exports = DocumentUpdaterController = { `doc-subscribed-${doc_id}` ) }) - roomEvents.on('doc-empty', (doc_id) => - rclientSubList.map((rclient) => + roomEvents.on('doc-empty', doc_id => + rclientSubList.map(rclient => ChannelManager.unsubscribe(rclient, 'applied-ops', doc_id) ) ) @@ -117,7 +117,7 @@ module.exports = DocumentUpdaterController = { doc_id, version: update.v, source: update.meta && update.meta.source, - socketIoClients: clientList.map((client) => client.id) + socketIoClients: clientList.map(client => client.id), }, 'distributing updates to clients' ) @@ -131,7 +131,7 @@ module.exports = DocumentUpdaterController = { { doc_id, version: update.v, - source: update.meta.source + source: update.meta.source, }, 'distributing update to sender' ) @@ -143,7 +143,7 @@ module.exports = DocumentUpdaterController = { doc_id, version: update.v, source: update.meta.source, - client_id: client.id + client_id: client.id, }, 'distributing update to collaborator' ) @@ -156,7 +156,7 @@ module.exports = DocumentUpdaterController = { logger.log( { doc_id, - socketIoClients: clientList.map((client) => client.id) + socketIoClients: clientList.map(client => client.id), }, 'discarded duplicate clients' ) @@ -172,5 +172,5 @@ module.exports = DocumentUpdaterController = { client.emit('otUpdateError', error, message) client.disconnect() } - } + }, } diff --git a/services/real-time/app/js/DocumentUpdaterManager.js b/services/real-time/app/js/DocumentUpdaterManager.js index 715d1a70d3..9f01ed73e3 100644 --- a/services/real-time/app/js/DocumentUpdaterManager.js +++ b/services/real-time/app/js/DocumentUpdaterManager.js @@ -11,7 +11,7 @@ const { ClientRequestedMissingOpsError, DocumentUpdaterRequestFailedError, NullBytesInOpError, - UpdateTooLargeError + UpdateTooLargeError, } = require('./Errors') const rclient = require('@overleaf/redis-wrapper').createClient( @@ -105,7 +105,7 @@ const DocumentUpdaterManager = { 'dupIfSource', 'meta', 'lastV', - 'hash' + 'hash', ] change = _.pick(change, allowedKeys) const jsonChange = JSON.stringify(change) @@ -125,24 +125,26 @@ const DocumentUpdaterManager = { const doc_key = `${project_id}:${doc_id}` // Push onto pendingUpdates for doc_id first, because once the doc updater // gets an entry on pending-updates-list, it starts processing. - rclient.rpush(Keys.pendingUpdates({ doc_id }), jsonChange, function ( - error - ) { - if (error) { - error = new OError('error pushing update into redis').withCause(error) - return callback(error) - } - const queueKey = DocumentUpdaterManager._getPendingUpdateListKey() - rclient.rpush(queueKey, doc_key, function (error) { + rclient.rpush( + Keys.pendingUpdates({ doc_id }), + jsonChange, + function (error) { if (error) { - error = new OError('error pushing doc_id into redis') - .withInfo({ queueKey }) - .withCause(error) + error = new OError('error pushing update into redis').withCause(error) + return callback(error) } - callback(error) - }) - }) - } + const queueKey = DocumentUpdaterManager._getPendingUpdateListKey() + rclient.rpush(queueKey, doc_key, function (error) { + if (error) { + error = new OError('error pushing doc_id into redis') + .withInfo({ queueKey }) + .withCause(error) + } + callback(error) + }) + } + ) + }, } module.exports = DocumentUpdaterManager diff --git a/services/real-time/app/js/DrainManager.js b/services/real-time/app/js/DrainManager.js index 06885f1d28..5f9c192afd 100644 --- a/services/real-time/app/js/DrainManager.js +++ b/services/real-time/app/js/DrainManager.js @@ -55,5 +55,5 @@ module.exports = { return true } return false - } + }, } diff --git a/services/real-time/app/js/Errors.js b/services/real-time/app/js/Errors.js index 562c9888ab..a5177be737 100644 --- a/services/real-time/app/js/Errors.js +++ b/services/real-time/app/js/Errors.js @@ -3,7 +3,7 @@ const OError = require('@overleaf/o-error') class ClientRequestedMissingOpsError extends OError { constructor(statusCode) { super('doc updater could not load requested ops', { - statusCode + statusCode, }) } } @@ -24,7 +24,7 @@ class DataTooLargeToParseError extends OError { constructor(data) { super('data too large to parse', { head: data.slice(0, 1024), - length: data.length + length: data.length, }) } } @@ -33,7 +33,7 @@ class DocumentUpdaterRequestFailedError extends OError { constructor(action, statusCode) { super('doc updater returned a non-success status code', { action, - statusCode + statusCode, }) } } @@ -99,5 +99,5 @@ module.exports = { NullBytesInOpError, UnexpectedArgumentsError, UpdateTooLargeError, - WebApiRequestFailedError + WebApiRequestFailedError, } diff --git a/services/real-time/app/js/EventLogger.js b/services/real-time/app/js/EventLogger.js index e6baffaa50..21c3460865 100644 --- a/services/real-time/app/js/EventLogger.js +++ b/services/real-time/app/js/EventLogger.js @@ -80,5 +80,5 @@ module.exports = EventLogger = { delete EVENT_LOG_TIMESTAMP[key] } }) - } + }, } diff --git a/services/real-time/app/js/HttpApiController.js b/services/real-time/app/js/HttpApiController.js index dceb294310..625bbbf739 100644 --- a/services/real-time/app/js/HttpApiController.js +++ b/services/real-time/app/js/HttpApiController.js @@ -48,5 +48,5 @@ module.exports = { logger.warn({ client_id }, 'api: requesting client disconnect') client.on('disconnect', () => res.sendStatus(204)) client.disconnect() - } + }, } diff --git a/services/real-time/app/js/HttpController.js b/services/real-time/app/js/HttpController.js index 68e5d4cd07..6e9d48cf63 100644 --- a/services/real-time/app/js/HttpController.js +++ b/services/real-time/app/js/HttpController.js @@ -17,7 +17,7 @@ module.exports = HttpController = { first_name, last_name, email, - connected_time + connected_time, } = ioClient.ol_context const client = { client_id, @@ -26,14 +26,14 @@ module.exports = HttpController = { first_name, last_name, email, - connected_time + connected_time, } client.rooms = Object.keys(ioClient.manager.roomClients[client_id] || {}) // drop the namespace - .filter((room) => room !== '') + .filter(room => room !== '') // room names are composed as '/' and the default // namespace is empty (see comments in RoomManager), just drop the '/' - .map((fullRoomPath) => fullRoomPath.slice(1)) + .map(fullRoomPath => fullRoomPath.slice(1)) return client }, @@ -53,5 +53,5 @@ module.exports = HttpController = { return } res.json(HttpController._getConnectedClientView(ioClient)) - } + }, } diff --git a/services/real-time/app/js/RedisClientManager.js b/services/real-time/app/js/RedisClientManager.js index c2070d3ad5..226bf91cfd 100644 --- a/services/real-time/app/js/RedisClientManager.js +++ b/services/real-time/app/js/RedisClientManager.js @@ -4,7 +4,7 @@ const logger = require('logger-sharelatex') module.exports = { createClientList(...configs) { // create a dynamic list of redis clients, excluding any configurations which are not defined - return configs.filter(Boolean).map((x) => { + return configs.filter(Boolean).map(x => { const redisType = x.cluster ? 'cluster' : x.sentinels @@ -15,5 +15,5 @@ module.exports = { logger.log({ redis: redisType }, 'creating redis client') return redis.createClient(x) }) - } + }, } diff --git a/services/real-time/app/js/RoomManager.js b/services/real-time/app/js/RoomManager.js index c1e03635a6..e030b60490 100644 --- a/services/real-time/app/js/RoomManager.js +++ b/services/real-time/app/js/RoomManager.js @@ -49,7 +49,7 @@ module.exports = { emitOnCompletion(promiseList, eventName) { Promise.all(promiseList) .then(() => RoomEvents.emit(eventName)) - .catch((err) => RoomEvents.emit(eventName, err)) + .catch(err => RoomEvents.emit(eventName, err)) }, eventSource() { @@ -150,15 +150,15 @@ module.exports = { return ( Object.keys(rooms) // drop the namespace - .filter((room) => room !== '') + .filter(room => room !== '') // room names are composed as '/' and the default // namespace is empty (see comments above), just drop the '/' - .map((fullRoomPath) => fullRoomPath.slice(1)) + .map(fullRoomPath => fullRoomPath.slice(1)) ) }, _clientAlreadyInRoom(client, room) { const rooms = client.manager.roomClients[client.id] || {} return !!rooms['/' + room] - } + }, } diff --git a/services/real-time/app/js/Router.js b/services/real-time/app/js/Router.js index 85b48eaf66..eff8dc7b1a 100644 --- a/services/real-time/app/js/Router.js +++ b/services/real-time/app/js/Router.js @@ -52,7 +52,7 @@ module.exports = Router = { [ 'not authorized', 'joinLeaveEpoch mismatch', - 'doc updater could not load requested ops' + 'doc updater could not load requested ops', ].includes(error.message) ) { logger.warn(attrs, error.message) @@ -62,7 +62,7 @@ module.exports = Router = { logger.error(attrs, `server side error in ${method}`) // Don't return raw error to prevent leaking server side info const serializedError = { - message: 'Something went wrong in real-time service' + message: 'Something went wrong in real-time service', } callback(serializedError) } @@ -201,7 +201,7 @@ module.exports = Router = { if (err) { Router._handleError(callback, err, client, 'joinProject', { project_id: data.project_id, - user_id: user._id + user_id: user._id, }) } else { callback(null, ...args) @@ -263,7 +263,7 @@ module.exports = Router = { if (err) { Router._handleError(callback, err, client, 'joinDoc', { doc_id, - fromVersion + fromVersion, }) } else { callback(null, ...args) @@ -280,7 +280,7 @@ module.exports = Router = { WebsocketController.leaveDoc(client, doc_id, function (err, ...args) { if (err) { Router._handleError(callback, err, client, 'leaveDoc', { - doc_id + doc_id, }) } else { callback(null, ...args) @@ -311,36 +311,38 @@ module.exports = Router = { }) }) - client.on('clientTracking.updatePosition', function ( - cursorData, - callback - ) { - if (!callback) { - callback = function () {} - } - if (typeof callback !== 'function') { - return Router._handleInvalidArguments( + client.on( + 'clientTracking.updatePosition', + function (cursorData, callback) { + if (!callback) { + callback = function () {} + } + if (typeof callback !== 'function') { + return Router._handleInvalidArguments( + client, + 'clientTracking.updatePosition', + arguments + ) + } + + WebsocketController.updateClientPosition( client, - 'clientTracking.updatePosition', - arguments + cursorData, + function (err) { + if (err) { + Router._handleError( + callback, + err, + client, + 'clientTracking.updatePosition' + ) + } else { + callback() + } + } ) } - - WebsocketController.updateClientPosition(client, cursorData, function ( - err - ) { - if (err) { - Router._handleError( - callback, - err, - client, - 'clientTracking.updatePosition' - ) - } else { - callback() - } - }) - }) + ) client.on('applyOtUpdate', function (doc_id, update, callback) { if (typeof callback !== 'function') { @@ -351,19 +353,22 @@ module.exports = Router = { ) } - WebsocketController.applyOtUpdate(client, doc_id, update, function ( - err - ) { - if (err) { - Router._handleError(callback, err, client, 'applyOtUpdate', { - doc_id, - update - }) - } else { - callback() + WebsocketController.applyOtUpdate( + client, + doc_id, + update, + function (err) { + if (err) { + Router._handleError(callback, err, client, 'applyOtUpdate', { + doc_id, + update, + }) + } else { + callback() + } } - }) + ) }) }) - } + }, } diff --git a/services/real-time/app/js/SafeJsonParse.js b/services/real-time/app/js/SafeJsonParse.js index b66f032ec6..bc7a6bed10 100644 --- a/services/real-time/app/js/SafeJsonParse.js +++ b/services/real-time/app/js/SafeJsonParse.js @@ -13,5 +13,5 @@ module.exports = { return callback(e) } callback(null, parsed) - } + }, } diff --git a/services/real-time/app/js/SessionSockets.js b/services/real-time/app/js/SessionSockets.js index f19147925a..b8ccde2ea8 100644 --- a/services/real-time/app/js/SessionSockets.js +++ b/services/real-time/app/js/SessionSockets.js @@ -20,7 +20,7 @@ module.exports = function (io, sessionStore, cookieParser, cookieName) { sessionStore.get(sessionId, function (error, session) { if (error) { OError.tag(error, 'error getting session from sessionStore', { - sessionId + sessionId, }) return next(error, socket) } diff --git a/services/real-time/app/js/WebApiManager.js b/services/real-time/app/js/WebApiManager.js index c5ac369149..a51610ee77 100644 --- a/services/real-time/app/js/WebApiManager.js +++ b/services/real-time/app/js/WebApiManager.js @@ -9,7 +9,7 @@ const { CodedError, CorruptedJoinProjectResponseError, NotAuthorizedError, - WebApiRequestFailedError + WebApiRequestFailedError, } = require('./Errors') module.exports = { @@ -28,11 +28,11 @@ module.exports = { auth: { user: settings.apis.web.user, pass: settings.apis.web.pass, - sendImmediately: true + sendImmediately: true, }, json: true, jar: false, - headers + headers, }, function (error, response, data) { if (error) { @@ -65,5 +65,5 @@ module.exports = { } } ) - } + }, } diff --git a/services/real-time/app/js/WebsocketController.js b/services/real-time/app/js/WebsocketController.js index bdf27d6967..c1ea8d232c 100644 --- a/services/real-time/app/js/WebsocketController.js +++ b/services/real-time/app/js/WebsocketController.js @@ -13,7 +13,7 @@ const RoomManager = require('./RoomManager') const { JoinLeaveEpochMismatchError, NotAuthorizedError, - NotJoinedError + NotJoinedError, } = require('./Errors') let WebsocketController @@ -26,7 +26,7 @@ module.exports = WebsocketController = { joinProject(client, user, project_id, callback) { if (client.disconnected) { metrics.inc('editor.join-project.disconnected', 1, { - status: 'immediately' + status: 'immediately', }) return callback() } @@ -37,71 +37,70 @@ module.exports = WebsocketController = { 'user joining project' ) metrics.inc('editor.join-project', 1, { status: client.transport }) - WebApiManager.joinProject(project_id, user, function ( - error, - project, - privilegeLevel, - isRestrictedUser - ) { - if (error) { - return callback(error) - } - if (client.disconnected) { - metrics.inc('editor.join-project.disconnected', 1, { - status: 'after-web-api-call' - }) - return callback() - } - - if (!privilegeLevel) { - return callback(new NotAuthorizedError()) - } - - client.ol_context = {} - client.ol_context.privilege_level = privilegeLevel - client.ol_context.user_id = user_id - client.ol_context.project_id = project_id - client.ol_context.owner_id = project.owner && project.owner._id - client.ol_context.first_name = user.first_name - client.ol_context.last_name = user.last_name - client.ol_context.email = user.email - client.ol_context.connected_time = new Date() - client.ol_context.signup_date = user.signUpDate - client.ol_context.login_count = user.loginCount - client.ol_context.is_restricted_user = !!isRestrictedUser - - RoomManager.joinProject(client, project_id, function (err) { - if (err) { - return callback(err) + WebApiManager.joinProject( + project_id, + user, + function (error, project, privilegeLevel, isRestrictedUser) { + if (error) { + return callback(error) + } + if (client.disconnected) { + metrics.inc('editor.join-project.disconnected', 1, { + status: 'after-web-api-call', + }) + return callback() } - logger.log( - { user_id, project_id, client_id: client.id }, - 'user joined project' - ) - callback( - null, - project, - privilegeLevel, - WebsocketController.PROTOCOL_VERSION - ) - }) - // No need to block for setting the user as connected in the cursor tracking - ConnectedUsersManager.updateUserPosition( - project_id, - client.publicId, - user, - null, - function (err) { + if (!privilegeLevel) { + return callback(new NotAuthorizedError()) + } + + client.ol_context = {} + client.ol_context.privilege_level = privilegeLevel + client.ol_context.user_id = user_id + client.ol_context.project_id = project_id + client.ol_context.owner_id = project.owner && project.owner._id + client.ol_context.first_name = user.first_name + client.ol_context.last_name = user.last_name + client.ol_context.email = user.email + client.ol_context.connected_time = new Date() + client.ol_context.signup_date = user.signUpDate + client.ol_context.login_count = user.loginCount + client.ol_context.is_restricted_user = !!isRestrictedUser + + RoomManager.joinProject(client, project_id, function (err) { if (err) { - logger.warn( - { err, project_id, user_id, client_id: client.id }, - 'background cursor update failed' - ) + return callback(err) } - } - ) - }) + logger.log( + { user_id, project_id, client_id: client.id }, + 'user joined project' + ) + callback( + null, + project, + privilegeLevel, + WebsocketController.PROTOCOL_VERSION + ) + }) + + // No need to block for setting the user as connected in the cursor tracking + ConnectedUsersManager.updateUserPosition( + project_id, + client.publicId, + user, + null, + function (err) { + if (err) { + logger.warn( + { err, project_id, user_id, client_id: client.id }, + 'background cursor update failed' + ) + } + } + ) + } + ) }, // We want to flush a project if there are no more (local) connected clients @@ -177,110 +176,112 @@ module.exports = WebsocketController = { 'client joining doc' ) - WebsocketController._assertClientAuthorization(client, doc_id, function ( - error - ) { - if (error) { - return callback(error) - } - if (client.disconnected) { - metrics.inc('editor.join-doc.disconnected', 1, { - status: 'after-client-auth-check' - }) - // the client will not read the response anyways - return callback() - } - if (joinLeaveEpoch !== client.joinLeaveEpoch) { - // another joinDoc or leaveDoc rpc overtook us - return callback(new JoinLeaveEpochMismatchError()) - } - // ensure the per-doc applied-ops channel is subscribed before sending the - // doc to the client, so that no events are missed. - RoomManager.joinDoc(client, doc_id, function (error) { + WebsocketController._assertClientAuthorization( + client, + doc_id, + function (error) { if (error) { return callback(error) } if (client.disconnected) { metrics.inc('editor.join-doc.disconnected', 1, { - status: 'after-joining-room' + status: 'after-client-auth-check', }) // the client will not read the response anyways return callback() } - - DocumentUpdaterManager.getDocument( - project_id, - doc_id, - fromVersion, - function (error, lines, version, ranges, ops) { - if (error) { - return callback(error) - } - if (client.disconnected) { - metrics.inc('editor.join-doc.disconnected', 1, { - status: 'after-doc-updater-call' - }) - // the client will not read the response anyways - return callback() - } - - if (is_restricted_user && ranges && ranges.comments) { - ranges.comments = [] - } - - // Encode any binary bits of data so it can go via WebSockets - // See http://ecmanaut.blogspot.co.uk/2006/07/encoding-decoding-utf8-in-javascript.html - const encodeForWebsockets = (text) => - unescape(encodeURIComponent(text)) - const escapedLines = [] - for (let line of lines) { - try { - line = encodeForWebsockets(line) - } catch (err) { - OError.tag(err, 'error encoding line uri component', { line }) - return callback(err) - } - escapedLines.push(line) - } - if (options.encodeRanges) { - try { - for (const comment of (ranges && ranges.comments) || []) { - if (comment.op.c) { - comment.op.c = encodeForWebsockets(comment.op.c) - } - } - for (const change of (ranges && ranges.changes) || []) { - if (change.op.i) { - change.op.i = encodeForWebsockets(change.op.i) - } - if (change.op.d) { - change.op.d = encodeForWebsockets(change.op.d) - } - } - } catch (err) { - OError.tag(err, 'error encoding range uri component', { - ranges - }) - return callback(err) - } - } - - AuthorizationManager.addAccessToDoc(client, doc_id, () => {}) - logger.log( - { - user_id, - project_id, - doc_id, - fromVersion, - client_id: client.id - }, - 'client joined doc' - ) - callback(null, escapedLines, version, ops, ranges) + if (joinLeaveEpoch !== client.joinLeaveEpoch) { + // another joinDoc or leaveDoc rpc overtook us + return callback(new JoinLeaveEpochMismatchError()) + } + // ensure the per-doc applied-ops channel is subscribed before sending the + // doc to the client, so that no events are missed. + RoomManager.joinDoc(client, doc_id, function (error) { + if (error) { + return callback(error) } - ) - }) - }) + if (client.disconnected) { + metrics.inc('editor.join-doc.disconnected', 1, { + status: 'after-joining-room', + }) + // the client will not read the response anyways + return callback() + } + + DocumentUpdaterManager.getDocument( + project_id, + doc_id, + fromVersion, + function (error, lines, version, ranges, ops) { + if (error) { + return callback(error) + } + if (client.disconnected) { + metrics.inc('editor.join-doc.disconnected', 1, { + status: 'after-doc-updater-call', + }) + // the client will not read the response anyways + return callback() + } + + if (is_restricted_user && ranges && ranges.comments) { + ranges.comments = [] + } + + // Encode any binary bits of data so it can go via WebSockets + // See http://ecmanaut.blogspot.co.uk/2006/07/encoding-decoding-utf8-in-javascript.html + const encodeForWebsockets = text => + unescape(encodeURIComponent(text)) + const escapedLines = [] + for (let line of lines) { + try { + line = encodeForWebsockets(line) + } catch (err) { + OError.tag(err, 'error encoding line uri component', { line }) + return callback(err) + } + escapedLines.push(line) + } + if (options.encodeRanges) { + try { + for (const comment of (ranges && ranges.comments) || []) { + if (comment.op.c) { + comment.op.c = encodeForWebsockets(comment.op.c) + } + } + for (const change of (ranges && ranges.changes) || []) { + if (change.op.i) { + change.op.i = encodeForWebsockets(change.op.i) + } + if (change.op.d) { + change.op.d = encodeForWebsockets(change.op.d) + } + } + } catch (err) { + OError.tag(err, 'error encoding range uri component', { + ranges, + }) + return callback(err) + } + } + + AuthorizationManager.addAccessToDoc(client, doc_id, () => {}) + logger.log( + { + user_id, + project_id, + doc_id, + fromVersion, + client_id: client.id, + }, + 'client joined doc' + ) + callback(null, escapedLines, version, ops, ranges) + } + ) + }) + } + ) }, _assertClientAuthorization(client, doc_id, callback) { @@ -297,16 +298,18 @@ module.exports = WebsocketController = { if (error) { // No cached access, check docupdater const { project_id } = client.ol_context - DocumentUpdaterManager.checkDocument(project_id, doc_id, function ( - error - ) { - if (error) { - return callback(error) - } else { - // Success - AuthorizationManager.addAccessToDoc(client, doc_id, callback) + DocumentUpdaterManager.checkDocument( + project_id, + doc_id, + function (error) { + if (error) { + return callback(error) + } else { + // Success + AuthorizationManager.addAccessToDoc(client, doc_id, callback) + } } - }) + ) } else { // Access already cached callback() @@ -339,15 +342,10 @@ module.exports = WebsocketController = { } metrics.inc('editor.update-client-position', 0.1, { - status: client.transport + status: client.transport, }) - const { - project_id, - first_name, - last_name, - email, - user_id - } = client.ol_context + const { project_id, first_name, last_name, email, user_id } = + client.ol_context logger.log( { user_id, project_id, client_id: client.id, cursorData }, 'updating client position' @@ -388,12 +386,12 @@ module.exports = WebsocketController = { first_name, last_name, email, - _id: user_id + _id: user_id, }, { row: cursorData.row, column: cursorData.column, - doc_id: cursorData.doc_id + doc_id: cursorData.doc_id, }, callback ) @@ -433,19 +431,19 @@ module.exports = WebsocketController = { WebsocketLoadBalancer.emitToRoom(project_id, 'clientTracking.refresh') setTimeout( () => - ConnectedUsersManager.getConnectedUsers(project_id, function ( - error, - users - ) { - if (error) { - return callback(error) + ConnectedUsersManager.getConnectedUsers( + project_id, + function (error, users) { + if (error) { + return callback(error) + } + logger.log( + { user_id, project_id, client_id: client.id }, + 'got connected users' + ) + callback(null, users) } - logger.log( - { user_id, project_id, client_id: client.id }, - 'got connected users' - ) - callback(null, users) - }), + ), WebsocketController.CLIENT_REFRESH_DELAY ) }) @@ -485,7 +483,7 @@ module.exports = WebsocketController = { doc_id, project_id, client_id: client.id, - version: update.v + version: update.v, }, 'sending update to doc updater' ) @@ -510,13 +508,13 @@ module.exports = WebsocketController = { const message = { project_id, doc_id, - error: 'update is too large' + error: 'update is too large', } setTimeout(function () { if (client.disconnected) { // skip the message broadcast, the client has moved on return metrics.inc('editor.doc-update.disconnected', 1, { - status: 'at-otUpdateError' + status: 'at-otUpdateError', }) } client.emit('otUpdateError', message.error, message) @@ -527,7 +525,7 @@ module.exports = WebsocketController = { if (error) { OError.tag(error, 'document was not available for update', { - version: update.v + version: update.v, }) client.disconnect() } @@ -571,5 +569,5 @@ module.exports = WebsocketController = { } } return true - } + }, } diff --git a/services/real-time/app/js/WebsocketLoadBalancer.js b/services/real-time/app/js/WebsocketLoadBalancer.js index 4f1a4935a7..f90f28c76b 100644 --- a/services/real-time/app/js/WebsocketLoadBalancer.js +++ b/services/real-time/app/js/WebsocketLoadBalancer.js @@ -19,7 +19,7 @@ const RESTRICTED_USER_MESSAGE_TYPE_PASS_LIST = [ 'reciveNewDoc', 'reciveNewFile', 'reciveNewFolder', - 'removeEntity' + 'removeEntity', ] let WebsocketLoadBalancer @@ -38,14 +38,14 @@ module.exports = WebsocketLoadBalancer = { const data = JSON.stringify({ room_id, message, - payload + payload, }) logger.log( { room_id, message, payload, length: data.length }, 'emitting to room' ) - this.rclientPubList.map((rclientPub) => + this.rclientPubList.map(rclientPub => ChannelManager.publish(rclientPub, 'editor-events', room_id, data) ) }, @@ -74,7 +74,7 @@ module.exports = WebsocketLoadBalancer = { handleRoomUpdates(rclientSubList) { const roomEvents = RoomManager.eventSource() roomEvents.on('project-active', function (project_id) { - const subscribePromises = rclientSubList.map((rclient) => + const subscribePromises = rclientSubList.map(rclient => ChannelManager.subscribe(rclient, 'editor-events', project_id) ) RoomManager.emitOnCompletion( @@ -82,8 +82,8 @@ module.exports = WebsocketLoadBalancer = { `project-subscribed-${project_id}` ) }) - roomEvents.on('project-empty', (project_id) => - rclientSubList.map((rclient) => + roomEvents.on('project-empty', project_id => + rclientSubList.map(rclient => ChannelManager.unsubscribe(rclient, 'editor-events', project_id) ) ) @@ -108,7 +108,7 @@ module.exports = WebsocketLoadBalancer = { message: message.message, room_id: message.room_id, message_id: message._id, - socketIoClients: clientList.map((client) => client.id) + socketIoClients: clientList.map(client => client.id), }, 'refreshing client list' ) @@ -127,15 +127,14 @@ module.exports = WebsocketLoadBalancer = { } } - const is_restricted_message = !RESTRICTED_USER_MESSAGE_TYPE_PASS_LIST.includes( - message.message - ) + const is_restricted_message = + !RESTRICTED_USER_MESSAGE_TYPE_PASS_LIST.includes(message.message) // send messages only to unique clients (due to duplicate entries in io.sockets.clients) const clientList = io.sockets .clients(message.room_id) .filter( - (client) => + client => !(is_restricted_message && client.ol_context.is_restricted_user) ) @@ -149,7 +148,7 @@ module.exports = WebsocketLoadBalancer = { message: message.message, room_id: message.room_id, message_id: message._id, - socketIoClients: clientList.map((client) => client.id) + socketIoClients: clientList.map(client => client.id), }, 'distributing event to clients' ) @@ -168,5 +167,5 @@ module.exports = WebsocketLoadBalancer = { HealthCheckManager.check(channel, message.key) } }) - } + }, } diff --git a/services/real-time/config/settings.defaults.js b/services/real-time/config/settings.defaults.js index c15ca57f6f..49b15e46fd 100644 --- a/services/real-time/config/settings.defaults.js +++ b/services/real-time/config/settings.defaults.js @@ -12,7 +12,7 @@ const settings = { process.env.PUBSUB_REDIS_MAX_RETRIES_PER_REQUEST || process.env.REDIS_MAX_RETRIES_PER_REQUEST || '20' - ) + ), }, realtime: { @@ -32,13 +32,13 @@ const settings = { }, connectedUser({ project_id, client_id }) { return `connected_user:{${project_id}}:${client_id}` - } + }, }, maxRetriesPerRequest: parseInt( process.env.REAL_TIME_REDIS_MAX_RETRIES_PER_REQUEST || process.env.REDIS_MAX_RETRIES_PER_REQUEST || '20' - ) + ), }, documentupdater: { @@ -55,13 +55,13 @@ const settings = { key_schema: { pendingUpdates({ doc_id }) { return `PendingUpdates:{${doc_id}}` - } + }, }, maxRetriesPerRequest: parseInt( process.env.DOC_UPDATER_REDIS_MAX_RETRIES_PER_REQUEST || process.env.REDIS_MAX_RETRIES_PER_REQUEST || '20' - ) + ), }, websessions: { @@ -73,8 +73,8 @@ const settings = { process.env.WEB_REDIS_MAX_RETRIES_PER_REQUEST || process.env.REDIS_MAX_RETRIES_PER_REQUEST || '20' - ) - } + ), + }, }, internal: { @@ -82,8 +82,8 @@ const settings = { port: 3026, host: process.env.LISTEN_ADDRESS || 'localhost', user: 'sharelatex', - pass: 'password' - } + pass: 'password', + }, }, apis: { @@ -92,19 +92,19 @@ const settings = { process.env.WEB_API_HOST || process.env.WEB_HOST || 'localhost' }:${process.env.WEB_API_PORT || process.env.WEB_PORT || 3000}`, user: process.env.WEB_API_USER || 'sharelatex', - pass: process.env.WEB_API_PASSWORD || 'password' + pass: process.env.WEB_API_PASSWORD || 'password', }, documentupdater: { url: `http://${ process.env.DOCUMENT_UPDATER_HOST || process.env.DOCUPDATER_HOST || 'localhost' - }:3003` - } + }:3003`, + }, }, security: { - sessionSecret: process.env.SESSION_SECRET || 'secret-please-change' + sessionSecret: process.env.SESSION_SECRET || 'secret-please-change', }, cookieName: process.env.COOKIE_NAME || 'sharelatex.sid', @@ -155,13 +155,13 @@ const settings = { deploymentFile: process.env.DEPLOYMENT_FILE, sentry: { - dsn: process.env.SENTRY_DSN + dsn: process.env.SENTRY_DSN, }, errors: { catchUncaughtErrors: true, - shutdownOnUncaughtError: true - } + shutdownOnUncaughtError: true, + }, } // console.log settings.redis diff --git a/services/real-time/config/settings.test.js b/services/real-time/config/settings.test.js index 7e2e0b24b6..9b426631b9 100644 --- a/services/real-time/config/settings.test.js +++ b/services/real-time/config/settings.test.js @@ -1,5 +1,5 @@ module.exports = { errors: { - catchUncaughtErrors: false - } + catchUncaughtErrors: false, + }, } diff --git a/services/real-time/test/acceptance/js/ApplyUpdateTests.js b/services/real-time/test/acceptance/js/ApplyUpdateTests.js index 785afa3bf4..bfa05d0b35 100644 --- a/services/real-time/test/acceptance/js/ApplyUpdateTests.js +++ b/services/real-time/test/acceptance/js/ApplyUpdateTests.js @@ -24,7 +24,7 @@ const rclient = redis.createClient(settings.redis.documentupdater) const redisSettings = settings.redis -const PENDING_UPDATES_LIST_KEYS = [0, 1, 2, 3, 4, 5, 6, 7, 8, 9].map((n) => { +const PENDING_UPDATES_LIST_KEYS = [0, 1, 2, 3, 4, 5, 6, 7, 8, 9].map(n => { let key = 'pending-updates-list' if (n !== 0) { key += `-${n}` @@ -33,10 +33,8 @@ const PENDING_UPDATES_LIST_KEYS = [0, 1, 2, 3, 4, 5, 6, 7, 8, 9].map((n) => { }) function getPendingUpdatesList(cb) { - Promise.all( - PENDING_UPDATES_LIST_KEYS.map((key) => rclient.lrange(key, 0, -1)) - ) - .then((results) => { + Promise.all(PENDING_UPDATES_LIST_KEYS.map(key => rclient.lrange(key, 0, -1))) + .then(results => { cb( null, results.reduce((acc, more) => { @@ -51,7 +49,7 @@ function getPendingUpdatesList(cb) { } function clearPendingUpdatesList(cb) { - Promise.all(PENDING_UPDATES_LIST_KEYS.map((key) => rclient.del(key))) + Promise.all(PENDING_UPDATES_LIST_KEYS.map(key => rclient.del(key))) .then(() => cb(null)) .catch(cb) } @@ -59,17 +57,17 @@ function clearPendingUpdatesList(cb) { describe('applyOtUpdate', function () { before(function () { return (this.update = { - op: [{ i: 'foo', p: 42 }] + op: [{ i: 'foo', p: 42 }], }) }) describe('when authorized', function () { before(function (done) { return async.series( [ - (cb) => { + cb => { return FixturesManager.setUpProject( { - privilegeLevel: 'readAndWrite' + privilegeLevel: 'readAndWrite', }, (e, { project_id, user_id }) => { this.project_id = project_id @@ -79,7 +77,7 @@ describe('applyOtUpdate', function () { ) }, - (cb) => { + cb => { return FixturesManager.setUpDoc( this.project_id, { lines: this.lines, version: this.version, ops: this.ops }, @@ -90,12 +88,12 @@ describe('applyOtUpdate', function () { ) }, - (cb) => { + cb => { this.client = RealTimeClient.connect() return this.client.on('connectionAccepted', cb) }, - (cb) => { + cb => { return this.client.emit( 'joinProject', { project_id: this.project_id }, @@ -103,18 +101,18 @@ describe('applyOtUpdate', function () { ) }, - (cb) => { + cb => { return this.client.emit('joinDoc', this.doc_id, cb) }, - (cb) => { + cb => { return this.client.emit( 'applyOtUpdate', this.doc_id, this.update, cb ) - } + }, ], done ) @@ -132,7 +130,7 @@ describe('applyOtUpdate', function () { it('should push the update into redis', function (done) { rclient.lrange( redisSettings.documentupdater.key_schema.pendingUpdates({ - doc_id: this.doc_id + doc_id: this.doc_id, }), 0, -1, @@ -142,7 +140,7 @@ describe('applyOtUpdate', function () { update.op.should.deep.equal(this.update.op) update.meta.should.deep.equal({ source: this.client.publicId, - user_id: this.user_id + user_id: this.user_id, }) return done() } @@ -153,20 +151,20 @@ describe('applyOtUpdate', function () { return after(function (done) { return async.series( [ - (cb) => clearPendingUpdatesList(cb), - (cb) => + cb => clearPendingUpdatesList(cb), + cb => rclient.del( 'DocsWithPendingUpdates', `${this.project_id}:${this.doc_id}`, cb ), - (cb) => + cb => rclient.del( redisSettings.documentupdater.key_schema.pendingUpdates( this.doc_id ), cb - ) + ), ], done ) @@ -178,15 +176,15 @@ describe('applyOtUpdate', function () { this.update = { op: { p: 12, - t: 'update is too large'.repeat(1024 * 400) // >7MB - } + t: 'update is too large'.repeat(1024 * 400), // >7MB + }, } return async.series( [ - (cb) => { + cb => { return FixturesManager.setUpProject( { - privilegeLevel: 'readAndWrite' + privilegeLevel: 'readAndWrite', }, (e, { project_id, user_id }) => { this.project_id = project_id @@ -196,7 +194,7 @@ describe('applyOtUpdate', function () { ) }, - (cb) => { + cb => { return FixturesManager.setUpDoc( this.project_id, { lines: this.lines, version: this.version, ops: this.ops }, @@ -207,15 +205,15 @@ describe('applyOtUpdate', function () { ) }, - (cb) => { + cb => { this.client = RealTimeClient.connect() this.client.on('connectionAccepted', cb) - return this.client.on('otUpdateError', (otUpdateError) => { + return this.client.on('otUpdateError', otUpdateError => { this.otUpdateError = otUpdateError }) }, - (cb) => { + cb => { return this.client.emit( 'joinProject', { project_id: this.project_id }, @@ -223,21 +221,21 @@ describe('applyOtUpdate', function () { ) }, - (cb) => { + cb => { return this.client.emit('joinDoc', this.doc_id, cb) }, - (cb) => { + cb => { return this.client.emit( 'applyOtUpdate', this.doc_id, this.update, - (error) => { + error => { this.error = error return cb() } ) - } + }, ], done ) @@ -264,7 +262,7 @@ describe('applyOtUpdate', function () { return it('should not put the update in redis', function (done) { rclient.llen( redisSettings.documentupdater.key_schema.pendingUpdates({ - doc_id: this.doc_id + doc_id: this.doc_id, }), (error, len) => { len.should.equal(0) @@ -279,10 +277,10 @@ describe('applyOtUpdate', function () { before(function (done) { return async.series( [ - (cb) => { + cb => { return FixturesManager.setUpProject( { - privilegeLevel: 'readOnly' + privilegeLevel: 'readOnly', }, (e, { project_id, user_id }) => { this.project_id = project_id @@ -292,7 +290,7 @@ describe('applyOtUpdate', function () { ) }, - (cb) => { + cb => { return FixturesManager.setUpDoc( this.project_id, { lines: this.lines, version: this.version, ops: this.ops }, @@ -303,12 +301,12 @@ describe('applyOtUpdate', function () { ) }, - (cb) => { + cb => { this.client = RealTimeClient.connect() return this.client.on('connectionAccepted', cb) }, - (cb) => { + cb => { return this.client.emit( 'joinProject', { project_id: this.project_id }, @@ -316,21 +314,21 @@ describe('applyOtUpdate', function () { ) }, - (cb) => { + cb => { return this.client.emit('joinDoc', this.doc_id, cb) }, - (cb) => { + cb => { return this.client.emit( 'applyOtUpdate', this.doc_id, this.update, - (error) => { + error => { this.error = error return cb() } ) - } + }, ], done ) @@ -350,7 +348,7 @@ describe('applyOtUpdate', function () { return it('should not put the update in redis', function (done) { rclient.llen( redisSettings.documentupdater.key_schema.pendingUpdates({ - doc_id: this.doc_id + doc_id: this.doc_id, }), (error, len) => { len.should.equal(0) @@ -364,14 +362,14 @@ describe('applyOtUpdate', function () { return describe('when authorized to read-only with a comment update', function () { before(function (done) { this.comment_update = { - op: [{ c: 'foo', p: 42 }] + op: [{ c: 'foo', p: 42 }], } return async.series( [ - (cb) => { + cb => { return FixturesManager.setUpProject( { - privilegeLevel: 'readOnly' + privilegeLevel: 'readOnly', }, (e, { project_id, user_id }) => { this.project_id = project_id @@ -381,7 +379,7 @@ describe('applyOtUpdate', function () { ) }, - (cb) => { + cb => { return FixturesManager.setUpDoc( this.project_id, { lines: this.lines, version: this.version, ops: this.ops }, @@ -392,12 +390,12 @@ describe('applyOtUpdate', function () { ) }, - (cb) => { + cb => { this.client = RealTimeClient.connect() return this.client.on('connectionAccepted', cb) }, - (cb) => { + cb => { return this.client.emit( 'joinProject', { project_id: this.project_id }, @@ -405,18 +403,18 @@ describe('applyOtUpdate', function () { ) }, - (cb) => { + cb => { return this.client.emit('joinDoc', this.doc_id, cb) }, - (cb) => { + cb => { return this.client.emit( 'applyOtUpdate', this.doc_id, this.comment_update, cb ) - } + }, ], done ) @@ -434,7 +432,7 @@ describe('applyOtUpdate', function () { it('should push the update into redis', function (done) { rclient.lrange( redisSettings.documentupdater.key_schema.pendingUpdates({ - doc_id: this.doc_id + doc_id: this.doc_id, }), 0, -1, @@ -444,7 +442,7 @@ describe('applyOtUpdate', function () { update.op.should.deep.equal(this.comment_update.op) update.meta.should.deep.equal({ source: this.client.publicId, - user_id: this.user_id + user_id: this.user_id, }) return done() } @@ -455,20 +453,20 @@ describe('applyOtUpdate', function () { return after(function (done) { return async.series( [ - (cb) => clearPendingUpdatesList(cb), - (cb) => + cb => clearPendingUpdatesList(cb), + cb => rclient.del( 'DocsWithPendingUpdates', `${this.project_id}:${this.doc_id}`, cb ), - (cb) => + cb => rclient.del( redisSettings.documentupdater.key_schema.pendingUpdates({ - doc_id: this.doc_id + doc_id: this.doc_id, }), cb - ) + ), ], done ) diff --git a/services/real-time/test/acceptance/js/ClientTrackingTests.js b/services/real-time/test/acceptance/js/ClientTrackingTests.js index 702b2288ab..0fc6c6cd5b 100644 --- a/services/real-time/test/acceptance/js/ClientTrackingTests.js +++ b/services/real-time/test/acceptance/js/ClientTrackingTests.js @@ -25,11 +25,11 @@ describe('clientTracking', function () { before(function (done) { return async.series( [ - (cb) => { + cb => { return FixturesManager.setUpProject( { privilegeLevel: 'owner', - project: { name: 'Test Project' } + project: { name: 'Test Project' }, }, (error, { user_id, project_id }) => { this.user_id = user_id @@ -39,7 +39,7 @@ describe('clientTracking', function () { ) }, - (cb) => { + cb => { return FixturesManager.setUpDoc( this.project_id, { lines: this.lines, version: this.version, ops: this.ops }, @@ -50,43 +50,43 @@ describe('clientTracking', function () { ) }, - (cb) => { + cb => { this.clientA = RealTimeClient.connect() return this.clientA.on('connectionAccepted', cb) }, - (cb) => { + cb => { this.clientB = RealTimeClient.connect() return this.clientB.on('connectionAccepted', cb) }, - (cb) => { + cb => { return this.clientA.emit( 'joinProject', { - project_id: this.project_id + project_id: this.project_id, }, cb ) }, - (cb) => { + cb => { return this.clientA.emit('joinDoc', this.doc_id, cb) }, - (cb) => { + cb => { return this.clientB.emit( 'joinProject', { - project_id: this.project_id + project_id: this.project_id, }, cb ) }, - (cb) => { + cb => { this.updates = [] - this.clientB.on('clientTracking.clientUpdated', (data) => { + this.clientB.on('clientTracking.clientUpdated', data => { return this.updates.push(data) }) @@ -95,16 +95,16 @@ describe('clientTracking', function () { { row: (this.row = 42), column: (this.column = 36), - doc_id: this.doc_id + doc_id: this.doc_id, }, - (error) => { + error => { if (error != null) { throw error } return setTimeout(cb, 300) } ) - } // Give the message a chance to reach client B. + }, // Give the message a chance to reach client B. ], done ) @@ -118,8 +118,8 @@ describe('clientTracking', function () { doc_id: this.doc_id, id: this.clientA.publicId, user_id: this.user_id, - name: 'Joe Bloggs' - } + name: 'Joe Bloggs', + }, ]) }) @@ -132,7 +132,7 @@ describe('clientTracking', function () { expect(user.cursorData).to.deep.equal({ row: this.row, column: this.column, - doc_id: this.doc_id + doc_id: this.doc_id, }) return done() } @@ -147,12 +147,12 @@ describe('clientTracking', function () { before(function (done) { return async.series( [ - (cb) => { + cb => { return FixturesManager.setUpProject( { privilegeLevel: 'owner', project: { name: 'Test Project' }, - publicAccess: 'readAndWrite' + publicAccess: 'readAndWrite', }, (error, { user_id, project_id }) => { this.user_id = user_id @@ -162,7 +162,7 @@ describe('clientTracking', function () { ) }, - (cb) => { + cb => { return FixturesManager.setUpDoc( this.project_id, { lines: this.lines, version: this.version, ops: this.ops }, @@ -173,47 +173,47 @@ describe('clientTracking', function () { ) }, - (cb) => { + cb => { this.clientA = RealTimeClient.connect() return this.clientA.on('connectionAccepted', cb) }, - (cb) => { + cb => { return this.clientA.emit( 'joinProject', { - project_id: this.project_id + project_id: this.project_id, }, cb ) }, - (cb) => { + cb => { return RealTimeClient.setSession({}, cb) }, - (cb) => { + cb => { this.anonymous = RealTimeClient.connect() return this.anonymous.on('connectionAccepted', cb) }, - (cb) => { + cb => { return this.anonymous.emit( 'joinProject', { - project_id: this.project_id + project_id: this.project_id, }, cb ) }, - (cb) => { + cb => { return this.anonymous.emit('joinDoc', this.doc_id, cb) }, - (cb) => { + cb => { this.updates = [] - this.clientA.on('clientTracking.clientUpdated', (data) => { + this.clientA.on('clientTracking.clientUpdated', data => { return this.updates.push(data) }) @@ -222,16 +222,16 @@ describe('clientTracking', function () { { row: (this.row = 42), column: (this.column = 36), - doc_id: this.doc_id + doc_id: this.doc_id, }, - (error) => { + error => { if (error != null) { throw error } return setTimeout(cb, 300) } ) - } // Give the message a chance to reach client B. + }, // Give the message a chance to reach client B. ], done ) @@ -245,8 +245,8 @@ describe('clientTracking', function () { doc_id: this.doc_id, id: this.anonymous.publicId, user_id: 'anonymous-user', - name: '' - } + name: '', + }, ]) }) }) diff --git a/services/real-time/test/acceptance/js/DrainManagerTests.js b/services/real-time/test/acceptance/js/DrainManagerTests.js index f4d5636adb..0f4d67fb3d 100644 --- a/services/real-time/test/acceptance/js/DrainManagerTests.js +++ b/services/real-time/test/acceptance/js/DrainManagerTests.js @@ -24,8 +24,8 @@ const drain = function (rate, callback) { url: `http://localhost:3026/drain?rate=${rate}`, auth: { user: Settings.internal.realTime.user, - pass: Settings.internal.realTime.pass - } + pass: Settings.internal.realTime.pass, + }, }, (error, response, data) => callback(error, data) ) @@ -38,8 +38,8 @@ describe('DrainManagerTests', function () { { privilegeLevel: 'owner', project: { - name: 'Test Project' - } + name: 'Test Project', + }, }, (e, { project_id, user_id }) => { this.project_id = project_id @@ -71,17 +71,17 @@ describe('DrainManagerTests', function () { beforeEach(function (done) { return async.series( [ - (cb) => { + cb => { this.clientA = RealTimeClient.connect() return this.clientA.on('connectionAccepted', cb) }, - (cb) => { + cb => { this.clientB = RealTimeClient.connect() return this.clientB.on('connectionAccepted', cb) }, - (cb) => { + cb => { return this.clientA.emit( 'joinProject', { project_id: this.project_id }, @@ -89,13 +89,13 @@ describe('DrainManagerTests', function () { ) }, - (cb) => { + cb => { return this.clientB.emit( 'joinProject', { project_id: this.project_id }, cb ) - } + }, ], done ) @@ -105,14 +105,14 @@ describe('DrainManagerTests', function () { beforeEach(function (done) { return async.parallel( [ - (cb) => { + cb => { return this.clientA.on('reconnectGracefully', cb) }, - (cb) => { + cb => { return this.clientB.on('reconnectGracefully', cb) }, - (cb) => drain(2, cb) + cb => drain(2, cb), ], done ) diff --git a/services/real-time/test/acceptance/js/EarlyDisconnect.js b/services/real-time/test/acceptance/js/EarlyDisconnect.js index cce2b318ec..14356f42ac 100644 --- a/services/real-time/test/acceptance/js/EarlyDisconnect.js +++ b/services/real-time/test/acceptance/js/EarlyDisconnect.js @@ -45,13 +45,13 @@ describe('EarlyDisconnect', function () { beforeEach(function (done) { return async.series( [ - (cb) => { + cb => { return FixturesManager.setUpProject( { privilegeLevel: 'owner', project: { - name: 'Test Project' - } + name: 'Test Project', + }, }, (e, { project_id, user_id }) => { this.project_id = project_id @@ -61,12 +61,12 @@ describe('EarlyDisconnect', function () { ) }, - (cb) => { + cb => { this.clientA = RealTimeClient.connect() return this.clientA.on('connectionAccepted', cb) }, - (cb) => { + cb => { this.clientA.emit( 'joinProject', { project_id: this.project_id }, @@ -77,10 +77,10 @@ describe('EarlyDisconnect', function () { return this.clientA.disconnect() }, - (cb) => { + cb => { // wait for joinDoc and subscribe return setTimeout(cb, 500) - } + }, ], done ) @@ -88,7 +88,7 @@ describe('EarlyDisconnect', function () { // we can force the race condition, there is no need to repeat too often return Array.from(Array.from({ length: 5 }).map((_, i) => i + 1)).map( - (attempt) => + attempt => it(`should not subscribe to the pub/sub channel anymore (race ${attempt})`, function (done) { rclient.pubsub('CHANNELS', (err, resp) => { if (err) { @@ -106,13 +106,13 @@ describe('EarlyDisconnect', function () { beforeEach(function (done) { return async.series( [ - (cb) => { + cb => { return FixturesManager.setUpProject( { privilegeLevel: 'owner', project: { - name: 'Test Project' - } + name: 'Test Project', + }, }, (e, { project_id, user_id }) => { this.project_id = project_id @@ -122,12 +122,12 @@ describe('EarlyDisconnect', function () { ) }, - (cb) => { + cb => { this.clientA = RealTimeClient.connect() return this.clientA.on('connectionAccepted', cb) }, - (cb) => { + cb => { return this.clientA.emit( 'joinProject', { project_id: this.project_id }, @@ -140,7 +140,7 @@ describe('EarlyDisconnect', function () { ) }, - (cb) => { + cb => { return FixturesManager.setUpDoc( this.project_id, { lines: this.lines, version: this.version, ops: this.ops }, @@ -151,17 +151,17 @@ describe('EarlyDisconnect', function () { ) }, - (cb) => { + cb => { this.clientA.emit('joinDoc', this.doc_id, () => {}) // disconnect before joinDoc completes this.clientA.on('disconnect', () => cb()) return this.clientA.disconnect() }, - (cb) => { + cb => { // wait for subscribe and unsubscribe return setTimeout(cb, 100) - } + }, ], done ) @@ -169,7 +169,7 @@ describe('EarlyDisconnect', function () { // we can not force the race condition, so we have to try many times return Array.from(Array.from({ length: 20 }).map((_, i) => i + 1)).map( - (attempt) => + attempt => it(`should not subscribe to the pub/sub channels anymore (race ${attempt})`, function (done) { rclient.pubsub('CHANNELS', (err, resp) => { if (err) { @@ -194,13 +194,13 @@ describe('EarlyDisconnect', function () { beforeEach(function (done) { return async.series( [ - (cb) => { + cb => { return FixturesManager.setUpProject( { privilegeLevel: 'owner', project: { - name: 'Test Project' - } + name: 'Test Project', + }, }, (e, { project_id, user_id }) => { this.project_id = project_id @@ -210,12 +210,12 @@ describe('EarlyDisconnect', function () { ) }, - (cb) => { + cb => { this.clientA = RealTimeClient.connect() return this.clientA.on('connectionAccepted', cb) }, - (cb) => { + cb => { return this.clientA.emit( 'joinProject', { project_id: this.project_id }, @@ -228,7 +228,7 @@ describe('EarlyDisconnect', function () { ) }, - (cb) => { + cb => { return FixturesManager.setUpDoc( this.project_id, { lines: this.lines, version: this.version, ops: this.ops }, @@ -239,17 +239,17 @@ describe('EarlyDisconnect', function () { ) }, - (cb) => { + cb => { return this.clientA.emit('joinDoc', this.doc_id, cb) }, - (cb) => { + cb => { this.clientA.emit( 'clientTracking.updatePosition', { row: 42, column: 36, - doc_id: this.doc_id + doc_id: this.doc_id, }, () => {} ) @@ -258,10 +258,10 @@ describe('EarlyDisconnect', function () { return this.clientA.disconnect() }, - (cb) => { + cb => { // wait for updateClientPosition return setTimeout(cb, 100) - } + }, ], done ) @@ -269,7 +269,7 @@ describe('EarlyDisconnect', function () { // we can not force the race condition, so we have to try many times return Array.from(Array.from({ length: 20 }).map((_, i) => i + 1)).map( - (attempt) => + attempt => it(`should not show the client as connected (race ${attempt})`, function (done) { rclientRT.smembers( KeysRT.clientsInProject({ project_id: this.project_id }), diff --git a/services/real-time/test/acceptance/js/HttpControllerTests.js b/services/real-time/test/acceptance/js/HttpControllerTests.js index 5d40a30def..8704c18135 100644 --- a/services/real-time/test/acceptance/js/HttpControllerTests.js +++ b/services/real-time/test/acceptance/js/HttpControllerTests.js @@ -11,7 +11,7 @@ const async = require('async') const { expect } = require('chai') const request = require('request').defaults({ - baseUrl: 'http://localhost:3026' + baseUrl: 'http://localhost:3026', }) const RealTimeClient = require('./helpers/RealTimeClient') @@ -24,7 +24,7 @@ describe('HttpControllerTests', function () { return request.get( { url: `/clients/${client_id}`, - json: true + json: true, }, (error, response, data) => { if (error) { @@ -41,10 +41,10 @@ describe('HttpControllerTests', function () { before(function (done) { return async.series( [ - (cb) => { + cb => { return FixturesManager.setUpProject( { - privilegeLevel: 'owner' + privilegeLevel: 'owner', }, (error, { project_id, user_id }) => { this.project_id = project_id @@ -54,7 +54,7 @@ describe('HttpControllerTests', function () { ) }, - (cb) => { + cb => { return FixturesManager.setUpDoc( this.project_id, {}, @@ -65,12 +65,12 @@ describe('HttpControllerTests', function () { ) }, - (cb) => { + cb => { this.client = RealTimeClient.connect() return this.client.on('connectionAccepted', cb) }, - (cb) => { + cb => { return this.client.emit( 'joinProject', { project_id: this.project_id }, @@ -78,9 +78,9 @@ describe('HttpControllerTests', function () { ) }, - (cb) => { + cb => { return this.client.emit('joinDoc', this.doc_id, cb) - } + }, ], done ) @@ -90,7 +90,7 @@ describe('HttpControllerTests', function () { return request.get( { url: `/clients/${this.client.socket.sessionid}`, - json: true + json: true, }, (error, response, data) => { if (error) { @@ -107,7 +107,7 @@ describe('HttpControllerTests', function () { last_name: 'Bloggs', project_id: this.project_id, user_id: this.user_id, - rooms: [this.project_id, this.doc_id] + rooms: [this.project_id, this.doc_id], }) return done() } diff --git a/services/real-time/test/acceptance/js/JoinDocTests.js b/services/real-time/test/acceptance/js/JoinDocTests.js index 936b8c8e04..abe8714cca 100644 --- a/services/real-time/test/acceptance/js/JoinDocTests.js +++ b/services/real-time/test/acceptance/js/JoinDocTests.js @@ -31,10 +31,10 @@ describe('joinDoc', function () { before(function (done) { return async.series( [ - (cb) => { + cb => { return FixturesManager.setUpProject( { - privilegeLevel: 'readAndWrite' + privilegeLevel: 'readAndWrite', }, (e, { project_id, user_id }) => { this.project_id = project_id @@ -44,14 +44,14 @@ describe('joinDoc', function () { ) }, - (cb) => { + cb => { return FixturesManager.setUpDoc( this.project_id, { lines: this.lines, version: this.version, ops: this.ops, - ranges: this.ranges + ranges: this.ranges, }, (e, { doc_id }) => { this.doc_id = doc_id @@ -60,12 +60,12 @@ describe('joinDoc', function () { ) }, - (cb) => { + cb => { this.client = RealTimeClient.connect() return this.client.on('connectionAccepted', cb) }, - (cb) => { + cb => { return this.client.emit( 'joinProject', { project_id: this.project_id }, @@ -73,7 +73,7 @@ describe('joinDoc', function () { ) }, - (cb) => { + cb => { return this.client.emit( 'joinDoc', this.doc_id, @@ -82,7 +82,7 @@ describe('joinDoc', function () { return cb(error) } ) - } + }, ], done ) @@ -99,7 +99,7 @@ describe('joinDoc', function () { this.lines, this.version, this.ops, - this.ranges + this.ranges, ]) }) @@ -118,10 +118,10 @@ describe('joinDoc', function () { before(function (done) { return async.series( [ - (cb) => { + cb => { return FixturesManager.setUpProject( { - privilegeLevel: 'readOnly' + privilegeLevel: 'readOnly', }, (e, { project_id, user_id }) => { this.project_id = project_id @@ -131,14 +131,14 @@ describe('joinDoc', function () { ) }, - (cb) => { + cb => { return FixturesManager.setUpDoc( this.project_id, { lines: this.lines, version: this.version, ops: this.ops, - ranges: this.ranges + ranges: this.ranges, }, (e, { doc_id }) => { this.doc_id = doc_id @@ -147,12 +147,12 @@ describe('joinDoc', function () { ) }, - (cb) => { + cb => { this.client = RealTimeClient.connect() return this.client.on('connectionAccepted', cb) }, - (cb) => { + cb => { return this.client.emit( 'joinProject', { project_id: this.project_id }, @@ -160,7 +160,7 @@ describe('joinDoc', function () { ) }, - (cb) => { + cb => { return this.client.emit( 'joinDoc', this.doc_id, @@ -169,7 +169,7 @@ describe('joinDoc', function () { return cb(error) } ) - } + }, ], done ) @@ -186,7 +186,7 @@ describe('joinDoc', function () { this.lines, this.version, this.ops, - this.ranges + this.ranges, ]) }) @@ -205,10 +205,10 @@ describe('joinDoc', function () { before(function (done) { return async.series( [ - (cb) => { + cb => { return FixturesManager.setUpProject( { - privilegeLevel: 'owner' + privilegeLevel: 'owner', }, (e, { project_id, user_id }) => { this.project_id = project_id @@ -218,14 +218,14 @@ describe('joinDoc', function () { ) }, - (cb) => { + cb => { return FixturesManager.setUpDoc( this.project_id, { lines: this.lines, version: this.version, ops: this.ops, - ranges: this.ranges + ranges: this.ranges, }, (e, { doc_id }) => { this.doc_id = doc_id @@ -234,12 +234,12 @@ describe('joinDoc', function () { ) }, - (cb) => { + cb => { this.client = RealTimeClient.connect() return this.client.on('connectionAccepted', cb) }, - (cb) => { + cb => { return this.client.emit( 'joinProject', { project_id: this.project_id }, @@ -247,7 +247,7 @@ describe('joinDoc', function () { ) }, - (cb) => { + cb => { return this.client.emit( 'joinDoc', this.doc_id, @@ -256,7 +256,7 @@ describe('joinDoc', function () { return cb(error) } ) - } + }, ], done ) @@ -273,7 +273,7 @@ describe('joinDoc', function () { this.lines, this.version, this.ops, - this.ranges + this.ranges, ]) }) @@ -297,10 +297,10 @@ describe('joinDoc', function () { this.fromVersion = 36 return async.series( [ - (cb) => { + cb => { return FixturesManager.setUpProject( { - privilegeLevel: 'readAndWrite' + privilegeLevel: 'readAndWrite', }, (e, { project_id, user_id }) => { this.project_id = project_id @@ -310,14 +310,14 @@ describe('joinDoc', function () { ) }, - (cb) => { + cb => { return FixturesManager.setUpDoc( this.project_id, { lines: this.lines, version: this.version, ops: this.ops, - ranges: this.ranges + ranges: this.ranges, }, (e, { doc_id }) => { this.doc_id = doc_id @@ -326,12 +326,12 @@ describe('joinDoc', function () { ) }, - (cb) => { + cb => { this.client = RealTimeClient.connect() return this.client.on('connectionAccepted', cb) }, - (cb) => { + cb => { return this.client.emit( 'joinProject', { project_id: this.project_id }, @@ -339,7 +339,7 @@ describe('joinDoc', function () { ) }, - (cb) => { + cb => { return this.client.emit( 'joinDoc', this.doc_id, @@ -349,7 +349,7 @@ describe('joinDoc', function () { return cb(error) } ) - } + }, ], done ) @@ -366,7 +366,7 @@ describe('joinDoc', function () { this.lines, this.version, this.ops, - this.ranges + this.ranges, ]) }) @@ -386,10 +386,10 @@ describe('joinDoc', function () { this.options = { encodeRanges: true } return async.series( [ - (cb) => { + cb => { return FixturesManager.setUpProject( { - privilegeLevel: 'readAndWrite' + privilegeLevel: 'readAndWrite', }, (e, { project_id, user_id }) => { this.project_id = project_id @@ -399,14 +399,14 @@ describe('joinDoc', function () { ) }, - (cb) => { + cb => { return FixturesManager.setUpDoc( this.project_id, { lines: this.lines, version: this.version, ops: this.ops, - ranges: this.ranges + ranges: this.ranges, }, (e, { doc_id }) => { this.doc_id = doc_id @@ -415,12 +415,12 @@ describe('joinDoc', function () { ) }, - (cb) => { + cb => { this.client = RealTimeClient.connect() return this.client.on('connectionAccepted', cb) }, - (cb) => { + cb => { return this.client.emit( 'joinProject', { project_id: this.project_id }, @@ -428,7 +428,7 @@ describe('joinDoc', function () { ) }, - (cb) => { + cb => { return this.client.emit( 'joinDoc', this.doc_id, @@ -438,7 +438,7 @@ describe('joinDoc', function () { return cb(error) } ) - } + }, ], done ) @@ -455,7 +455,7 @@ describe('joinDoc', function () { this.lines, this.version, this.ops, - this.ranges + this.ranges, ]) }) @@ -476,10 +476,10 @@ describe('joinDoc', function () { this.options = { encodeRanges: true } return async.series( [ - (cb) => { + cb => { return FixturesManager.setUpProject( { - privilegeLevel: 'readAndWrite' + privilegeLevel: 'readAndWrite', }, (e, { project_id, user_id }) => { this.project_id = project_id @@ -489,14 +489,14 @@ describe('joinDoc', function () { ) }, - (cb) => { + cb => { return FixturesManager.setUpDoc( this.project_id, { lines: this.lines, version: this.version, ops: this.ops, - ranges: this.ranges + ranges: this.ranges, }, (e, { doc_id }) => { this.doc_id = doc_id @@ -505,12 +505,12 @@ describe('joinDoc', function () { ) }, - (cb) => { + cb => { this.client = RealTimeClient.connect() return this.client.on('connectionAccepted', cb) }, - (cb) => { + cb => { return this.client.emit( 'joinProject', { project_id: this.project_id }, @@ -518,7 +518,7 @@ describe('joinDoc', function () { ) }, - (cb) => { + cb => { return this.client.emit( 'joinDoc', this.doc_id, @@ -529,7 +529,7 @@ describe('joinDoc', function () { return cb(error) } ) - } + }, ], done ) @@ -546,7 +546,7 @@ describe('joinDoc', function () { this.lines, this.version, this.ops, - this.ranges + this.ranges, ]) }) diff --git a/services/real-time/test/acceptance/js/JoinProjectTests.js b/services/real-time/test/acceptance/js/JoinProjectTests.js index 61ede26f87..63e46d9a3a 100644 --- a/services/real-time/test/acceptance/js/JoinProjectTests.js +++ b/services/real-time/test/acceptance/js/JoinProjectTests.js @@ -23,13 +23,13 @@ describe('joinProject', function () { before(function (done) { return async.series( [ - (cb) => { + cb => { return FixturesManager.setUpProject( { privilegeLevel: 'owner', project: { - name: 'Test Project' - } + name: 'Test Project', + }, }, (e, { project_id, user_id }) => { this.project_id = project_id @@ -39,12 +39,12 @@ describe('joinProject', function () { ) }, - (cb) => { + cb => { this.client = RealTimeClient.connect() return this.client.on('connectionAccepted', cb) }, - (cb) => { + cb => { return this.client.emit( 'joinProject', { project_id: this.project_id }, @@ -55,7 +55,7 @@ describe('joinProject', function () { return cb(error) } ) - } + }, ], done ) @@ -69,7 +69,7 @@ describe('joinProject', function () { it('should return the project', function () { return this.project.should.deep.equal({ - name: 'Test Project' + name: 'Test Project', }) }) @@ -118,13 +118,13 @@ describe('joinProject', function () { before(function (done) { return async.series( [ - (cb) => { + cb => { return FixturesManager.setUpProject( { privilegeLevel: null, project: { - name: 'Test Project' - } + name: 'Test Project', + }, }, (e, { project_id, user_id }) => { this.project_id = project_id @@ -134,12 +134,12 @@ describe('joinProject', function () { ) }, - (cb) => { + cb => { this.client = RealTimeClient.connect() return this.client.on('connectionAccepted', cb) }, - (cb) => { + cb => { return this.client.emit( 'joinProject', { project_id: this.project_id }, @@ -151,7 +151,7 @@ describe('joinProject', function () { return cb() } ) - } + }, ], done ) @@ -178,14 +178,14 @@ describe('joinProject', function () { before(function (done) { return async.series( [ - (cb) => { + cb => { return FixturesManager.setUpProject( { project_id: 'forbidden', privilegeLevel: 'owner', project: { - name: 'Test Project' - } + name: 'Test Project', + }, }, (e, { project_id, user_id }) => { this.project_id = project_id @@ -195,12 +195,12 @@ describe('joinProject', function () { ) }, - (cb) => { + cb => { this.client = RealTimeClient.connect() this.client.on('connectionAccepted', cb) }, - (cb) => { + cb => { this.client.emit( 'joinProject', { project_id: this.project_id }, @@ -212,7 +212,7 @@ describe('joinProject', function () { cb() } ) - } + }, ], done ) @@ -239,14 +239,14 @@ describe('joinProject', function () { before(function (done) { return async.series( [ - (cb) => { + cb => { return FixturesManager.setUpProject( { project_id: 'not-found', privilegeLevel: 'owner', project: { - name: 'Test Project' - } + name: 'Test Project', + }, }, (e, { project_id, user_id }) => { this.project_id = project_id @@ -256,12 +256,12 @@ describe('joinProject', function () { ) }, - (cb) => { + cb => { this.client = RealTimeClient.connect() this.client.on('connectionAccepted', cb) }, - (cb) => { + cb => { this.client.emit( 'joinProject', { project_id: this.project_id }, @@ -273,7 +273,7 @@ describe('joinProject', function () { cb() } ) - } + }, ], done ) @@ -300,21 +300,21 @@ describe('joinProject', function () { before(function (done) { return async.series( [ - (cb) => { + cb => { this.client = RealTimeClient.connect() return this.client.on('connectionAccepted', cb) }, - (cb) => { + cb => { return this.client.emit( 'joinProject', { project_id: 'rate-limited' }, - (error) => { + error => { this.error = error return cb() } ) - } + }, ], done ) diff --git a/services/real-time/test/acceptance/js/LeaveDocTests.js b/services/real-time/test/acceptance/js/LeaveDocTests.js index fc77da6314..61a3288176 100644 --- a/services/real-time/test/acceptance/js/LeaveDocTests.js +++ b/services/real-time/test/acceptance/js/LeaveDocTests.js @@ -44,10 +44,10 @@ describe('leaveDoc', function () { beforeEach(function (done) { return async.series( [ - (cb) => { + cb => { return FixturesManager.setUpProject( { - privilegeLevel: 'readAndWrite' + privilegeLevel: 'readAndWrite', }, (e, { project_id, user_id }) => { this.project_id = project_id @@ -57,7 +57,7 @@ describe('leaveDoc', function () { ) }, - (cb) => { + cb => { return FixturesManager.setUpDoc( this.project_id, { lines: this.lines, version: this.version, ops: this.ops }, @@ -68,12 +68,12 @@ describe('leaveDoc', function () { ) }, - (cb) => { + cb => { this.client = RealTimeClient.connect() return this.client.on('connectionAccepted', cb) }, - (cb) => { + cb => { return this.client.emit( 'joinProject', { project_id: this.project_id }, @@ -81,7 +81,7 @@ describe('leaveDoc', function () { ) }, - (cb) => { + cb => { return this.client.emit( 'joinDoc', this.doc_id, @@ -90,7 +90,7 @@ describe('leaveDoc', function () { return cb(error) } ) - } + }, ], done ) @@ -98,7 +98,7 @@ describe('leaveDoc', function () { describe('then leaving the doc', function () { beforeEach(function (done) { - return this.client.emit('leaveDoc', this.doc_id, (error) => { + return this.client.emit('leaveDoc', this.doc_id, error => { if (error != null) { throw error } @@ -123,7 +123,7 @@ describe('leaveDoc', function () { beforeEach(function (done) { this.client.emit('leaveDoc', this.doc_id, () => {}) this.client.emit('joinDoc', this.doc_id, () => {}) - return this.client.emit('leaveDoc', this.doc_id, (error) => { + return this.client.emit('leaveDoc', this.doc_id, error => { if (error != null) { throw error } @@ -154,7 +154,7 @@ describe('leaveDoc', function () { return describe('when sending a leaveDoc for a room the client has not joined ', function () { beforeEach(function (done) { - return this.client.emit('leaveDoc', this.other_doc_id, (error) => { + return this.client.emit('leaveDoc', this.other_doc_id, error => { if (error != null) { throw error } diff --git a/services/real-time/test/acceptance/js/LeaveProjectTests.js b/services/real-time/test/acceptance/js/LeaveProjectTests.js index 0d30c90c8f..9c71678087 100644 --- a/services/real-time/test/acceptance/js/LeaveProjectTests.js +++ b/services/real-time/test/acceptance/js/LeaveProjectTests.js @@ -30,13 +30,13 @@ describe('leaveProject', function () { before(function (done) { return async.series( [ - (cb) => { + cb => { return FixturesManager.setUpProject( { privilegeLevel: 'owner', project: { - name: 'Test Project' - } + name: 'Test Project', + }, }, (e, { project_id, user_id }) => { this.project_id = project_id @@ -46,25 +46,25 @@ describe('leaveProject', function () { ) }, - (cb) => { + cb => { this.clientA = RealTimeClient.connect() return this.clientA.on('connectionAccepted', cb) }, - (cb) => { + cb => { this.clientB = RealTimeClient.connect() this.clientB.on('connectionAccepted', cb) this.clientBDisconnectMessages = [] return this.clientB.on( 'clientTracking.clientDisconnected', - (data) => { + data => { return this.clientBDisconnectMessages.push(data) } ) }, - (cb) => { + cb => { return this.clientA.emit( 'joinProject', { project_id: this.project_id }, @@ -77,7 +77,7 @@ describe('leaveProject', function () { ) }, - (cb) => { + cb => { return this.clientB.emit( 'joinProject', { project_id: this.project_id }, @@ -90,7 +90,7 @@ describe('leaveProject', function () { ) }, - (cb) => { + cb => { return FixturesManager.setUpDoc( this.project_id, { lines: this.lines, version: this.version, ops: this.ops }, @@ -101,23 +101,23 @@ describe('leaveProject', function () { ) }, - (cb) => { + cb => { return this.clientA.emit('joinDoc', this.doc_id, cb) }, - (cb) => { + cb => { return this.clientB.emit('joinDoc', this.doc_id, cb) }, - (cb) => { + cb => { // leaveProject is called when the client disconnects this.clientA.on('disconnect', () => cb()) return this.clientA.disconnect() }, - (cb) => { + cb => { // The API waits a little while before flushing changes return setTimeout(done, 1000) - } + }, ], done ) @@ -125,7 +125,7 @@ describe('leaveProject', function () { it('should emit a disconnect message to the room', function () { return this.clientBDisconnectMessages.should.deep.equal([ - this.clientA.publicId + this.clientA.publicId, ]) }) @@ -176,13 +176,13 @@ describe('leaveProject', function () { before(function (done) { return async.series( [ - (cb) => { + cb => { return FixturesManager.setUpProject( { privilegeLevel: 'owner', project: { - name: 'Test Project' - } + name: 'Test Project', + }, }, (e, { project_id, user_id }) => { this.project_id = project_id @@ -192,12 +192,12 @@ describe('leaveProject', function () { ) }, - (cb) => { + cb => { this.clientA = RealTimeClient.connect() return this.clientA.on('connect', cb) }, - (cb) => { + cb => { return this.clientA.emit( 'joinProject', { project_id: this.project_id }, @@ -210,7 +210,7 @@ describe('leaveProject', function () { ) }, - (cb) => { + cb => { return FixturesManager.setUpDoc( this.project_id, { lines: this.lines, version: this.version, ops: this.ops }, @@ -220,20 +220,20 @@ describe('leaveProject', function () { } ) }, - (cb) => { + cb => { return this.clientA.emit('joinDoc', this.doc_id, cb) }, - (cb) => { + cb => { // leaveProject is called when the client disconnects this.clientA.on('disconnect', () => cb()) return this.clientA.disconnect() }, - (cb) => { + cb => { // The API waits a little while before flushing changes return setTimeout(done, 1000) - } + }, ], done ) diff --git a/services/real-time/test/acceptance/js/MatrixTests.js b/services/real-time/test/acceptance/js/MatrixTests.js index ee7686679a..690a61fc24 100644 --- a/services/real-time/test/acceptance/js/MatrixTests.js +++ b/services/real-time/test/acceptance/js/MatrixTests.js @@ -81,7 +81,7 @@ describe('MatrixTests', function () { privateClient.emit( 'joinProject', { project_id: privateProjectId }, - (err) => { + err => { if (err) return done(err) privateClient.emit('joinDoc', privateDocId, done) } @@ -94,7 +94,7 @@ describe('MatrixTests', function () { before(function setupReadWriteProject(done) { FixturesManager.setUpEditorSession( { - publicAccess: 'readAndWrite' + publicAccess: 'readAndWrite', }, (err, { project_id, doc_id }) => { readWriteProjectId = project_id @@ -107,13 +107,13 @@ describe('MatrixTests', function () { const USER_SETUP = { anonymous: { setup(cb) { - RealTimeClient.setSession({}, (err) => { + RealTimeClient.setSession({}, err => { if (err) return cb(err) cb(null, { - client: RealTimeClient.connect() + client: RealTimeClient.connect(), }) }) - } + }, }, registered: { @@ -124,18 +124,18 @@ describe('MatrixTests', function () { user: { _id: user_id, first_name: 'Joe', - last_name: 'Bloggs' - } + last_name: 'Bloggs', + }, }, - (err) => { + err => { if (err) return cb(err) cb(null, { user_id, - client: RealTimeClient.connect() + client: RealTimeClient.connect(), }) } ) - } + }, }, registeredWithOwnedProject: { @@ -148,16 +148,16 @@ describe('MatrixTests', function () { user_id, project_id, doc_id, - client: RealTimeClient.connect() + client: RealTimeClient.connect(), }) } ) }, - hasOwnProject: true - } + hasOwnProject: true, + }, } - Object.entries(USER_SETUP).forEach((level0) => { + Object.entries(USER_SETUP).forEach(level0 => { const [userDescription, userItem] = level0 let options, client @@ -166,46 +166,46 @@ describe('MatrixTests', function () { getActions(cb) { cb(null, []) }, - needsOwnProject: false + needsOwnProject: false, }, joinReadWriteProject: { getActions(cb) { cb(null, [ - { rpc: 'joinProject', args: [{ project_id: readWriteProjectId }] } + { rpc: 'joinProject', args: [{ project_id: readWriteProjectId }] }, ]) }, - needsOwnProject: false + needsOwnProject: false, }, joinReadWriteProjectAndDoc: { getActions(cb) { cb(null, [ { rpc: 'joinProject', args: [{ project_id: readWriteProjectId }] }, - { rpc: 'joinDoc', args: [readWriteDocId] } + { rpc: 'joinDoc', args: [readWriteDocId] }, ]) }, - needsOwnProject: false + needsOwnProject: false, }, joinOwnProject: { getActions(cb) { cb(null, [ - { rpc: 'joinProject', args: [{ project_id: options.project_id }] } + { rpc: 'joinProject', args: [{ project_id: options.project_id }] }, ]) }, - needsOwnProject: true + needsOwnProject: true, }, joinOwnProjectAndDoc: { getActions(cb) { cb(null, [ { rpc: 'joinProject', args: [{ project_id: options.project_id }] }, - { rpc: 'joinDoc', args: [options.doc_id] } + { rpc: 'joinDoc', args: [options.doc_id] }, ]) }, - needsOwnProject: true - } + needsOwnProject: true, + }, } function performActions(getActions, done) { @@ -245,13 +245,13 @@ describe('MatrixTests', function () { }) }) - Object.entries(SESSION_SETUP).forEach((level1) => { + Object.entries(SESSION_SETUP).forEach(level1 => { const [sessionSetupDescription, sessionSetupItem] = level1 const INVALID_REQUESTS = { noop: { getActions(cb) { cb(null, []) - } + }, }, joinProjectWithDocId: { @@ -260,16 +260,16 @@ describe('MatrixTests', function () { { rpc: 'joinProject', args: [{ project_id: privateDocId }], - fails: 1 - } + fails: 1, + }, ]) - } + }, }, joinDocWithDocId: { getActions(cb) { cb(null, [{ rpc: 'joinDoc', args: [privateDocId], fails: 1 }]) - } + }, }, joinProjectWithProjectId: { @@ -278,16 +278,16 @@ describe('MatrixTests', function () { { rpc: 'joinProject', args: [{ project_id: privateProjectId }], - fails: 1 - } + fails: 1, + }, ]) - } + }, }, joinDocWithProjectId: { getActions(cb) { cb(null, [{ rpc: 'joinDoc', args: [privateProjectId], fails: 1 }]) - } + }, }, joinProjectWithProjectIdThenJoinDocWithDocId: { @@ -296,12 +296,12 @@ describe('MatrixTests', function () { { rpc: 'joinProject', args: [{ project_id: privateProjectId }], - fails: 1 + fails: 1, }, - { rpc: 'joinDoc', args: [privateDocId], fails: 1 } + { rpc: 'joinDoc', args: [privateDocId], fails: 1 }, ]) - } - } + }, + }, } // skip some areas of the matrix @@ -314,7 +314,7 @@ describe('MatrixTests', function () { performActions(sessionSetupItem.getActions, done) }) - Object.entries(INVALID_REQUESTS).forEach((level2) => { + Object.entries(INVALID_REQUESTS).forEach(level2 => { const [InvalidRequestDescription, InvalidRequestItem] = level2 describe(InvalidRequestDescription, function () { beforeEach(function performInvalidRequests(done) { @@ -354,10 +354,10 @@ describe('MatrixTests', function () { meta: { source: privateClient.publicId }, v: 42, doc: privateDocId, - op: [{ i: 'foo', p: 50 }] - } + op: [{ i: 'foo', p: 50 }], + }, } - client.on('otUpdateApplied', (update) => { + client.on('otUpdateApplied', update => { receivedMessages.push(update) }) privateClient.once('otUpdateApplied', () => { @@ -378,7 +378,7 @@ describe('MatrixTests', function () { room_id: privateProjectId, message: 'removeEntity', payload: ['foo', 'convertDocToFile'], - _id: 'web:123' + _id: 'web:123', } client.on('removeEntity', (...args) => { receivedMessages.push(args) @@ -408,14 +408,14 @@ describe('MatrixTests', function () { v: 43, lastV: 42, doc: privateDocId, - op: [{ i: 'foo', p: 50 }] - } + op: [{ i: 'foo', p: 50 }], + }, } }) beforeEach(function sendAsUser(done) { const userUpdate = Object.assign({}, update, { - hash: 'user' + hash: 'user', }) client.emit( @@ -431,7 +431,7 @@ describe('MatrixTests', function () { beforeEach(function sendAsPrivateUserForReferenceOp(done) { const privateUpdate = Object.assign({}, update, { - hash: 'private' + hash: 'private', }) privateClient.emit( @@ -457,7 +457,7 @@ describe('MatrixTests', function () { expect( [ 'no project_id found on client', - 'not authorized' + 'not authorized', ].includes(receivedArgs[0].message) ).to.equal(true) }) diff --git a/services/real-time/test/acceptance/js/PubSubRace.js b/services/real-time/test/acceptance/js/PubSubRace.js index db97a3d00e..9563f71354 100644 --- a/services/real-time/test/acceptance/js/PubSubRace.js +++ b/services/real-time/test/acceptance/js/PubSubRace.js @@ -28,13 +28,13 @@ describe('PubSubRace', function () { before(function (done) { return async.series( [ - (cb) => { + cb => { return FixturesManager.setUpProject( { privilegeLevel: 'owner', project: { - name: 'Test Project' - } + name: 'Test Project', + }, }, (e, { project_id, user_id }) => { this.project_id = project_id @@ -44,12 +44,12 @@ describe('PubSubRace', function () { ) }, - (cb) => { + cb => { this.clientA = RealTimeClient.connect() return this.clientA.on('connect', cb) }, - (cb) => { + cb => { return this.clientA.emit( 'joinProject', { project_id: this.project_id }, @@ -62,7 +62,7 @@ describe('PubSubRace', function () { ) }, - (cb) => { + cb => { return FixturesManager.setUpDoc( this.project_id, { lines: this.lines, version: this.version, ops: this.ops }, @@ -73,16 +73,16 @@ describe('PubSubRace', function () { ) }, - (cb) => { + cb => { this.clientA.emit('joinDoc', this.doc_id, () => {}) // leave before joinDoc completes return this.clientA.emit('leaveDoc', this.doc_id, cb) }, - (cb) => { + cb => { // wait for subscribe and unsubscribe return setTimeout(cb, 100) - } + }, ], done ) @@ -104,13 +104,13 @@ describe('PubSubRace', function () { before(function (done) { return async.series( [ - (cb) => { + cb => { return FixturesManager.setUpProject( { privilegeLevel: 'owner', project: { - name: 'Test Project' - } + name: 'Test Project', + }, }, (e, { project_id, user_id }) => { this.project_id = project_id @@ -120,12 +120,12 @@ describe('PubSubRace', function () { ) }, - (cb) => { + cb => { this.clientA = RealTimeClient.connect() return this.clientA.on('connect', cb) }, - (cb) => { + cb => { return this.clientA.emit( 'joinProject', { project_id: this.project_id }, @@ -138,7 +138,7 @@ describe('PubSubRace', function () { ) }, - (cb) => { + cb => { return FixturesManager.setUpDoc( this.project_id, { lines: this.lines, version: this.version, ops: this.ops }, @@ -149,7 +149,7 @@ describe('PubSubRace', function () { ) }, - (cb) => { + cb => { this.clientA.emit('joinDoc', this.doc_id, () => {}) this.clientA.emit('leaveDoc', this.doc_id, () => {}) this.clientA.emit('joinDoc', this.doc_id, () => {}) @@ -162,10 +162,10 @@ describe('PubSubRace', function () { return this.clientA.emit('leaveDoc', this.doc_id, cb) }, - (cb) => { + cb => { // wait for subscribe and unsubscribe return setTimeout(cb, 100) - } + }, ], done ) @@ -187,13 +187,13 @@ describe('PubSubRace', function () { before(function (done) { return async.series( [ - (cb) => { + cb => { return FixturesManager.setUpProject( { privilegeLevel: 'owner', project: { - name: 'Test Project' - } + name: 'Test Project', + }, }, (e, { project_id, user_id }) => { this.project_id = project_id @@ -203,12 +203,12 @@ describe('PubSubRace', function () { ) }, - (cb) => { + cb => { this.clientA = RealTimeClient.connect() return this.clientA.on('connect', cb) }, - (cb) => { + cb => { return this.clientA.emit( 'joinProject', { project_id: this.project_id }, @@ -221,7 +221,7 @@ describe('PubSubRace', function () { ) }, - (cb) => { + cb => { return FixturesManager.setUpDoc( this.project_id, { lines: this.lines, version: this.version, ops: this.ops }, @@ -232,7 +232,7 @@ describe('PubSubRace', function () { ) }, - (cb) => { + cb => { this.clientA.emit('joinDoc', this.doc_id, () => {}) this.clientA.emit('leaveDoc', this.doc_id, () => {}) this.clientA.emit('joinDoc', this.doc_id, () => {}) @@ -244,10 +244,10 @@ describe('PubSubRace', function () { return this.clientA.emit('joinDoc', this.doc_id, cb) }, - (cb) => { + cb => { // wait for subscribe and unsubscribe return setTimeout(cb, 100) - } + }, ], done ) @@ -269,13 +269,13 @@ describe('PubSubRace', function () { before(function (done) { return async.series( [ - (cb) => { + cb => { return FixturesManager.setUpProject( { privilegeLevel: 'owner', project: { - name: 'Test Project' - } + name: 'Test Project', + }, }, (e, { project_id, user_id }) => { this.project_id = project_id @@ -285,12 +285,12 @@ describe('PubSubRace', function () { ) }, - (cb) => { + cb => { this.clientA = RealTimeClient.connect() return this.clientA.on('connect', cb) }, - (cb) => { + cb => { return this.clientA.emit( 'joinProject', { project_id: this.project_id }, @@ -303,7 +303,7 @@ describe('PubSubRace', function () { ) }, - (cb) => { + cb => { return FixturesManager.setUpDoc( this.project_id, { lines: this.lines, version: this.version, ops: this.ops }, @@ -314,7 +314,7 @@ describe('PubSubRace', function () { ) }, - (cb) => { + cb => { let joinDocCompleted = false this.clientA.emit( 'joinDoc', @@ -339,10 +339,10 @@ describe('PubSubRace', function () { ) }, - (cb) => { + cb => { // wait for subscribe and unsubscribe return setTimeout(cb, 100) - } + }, ], done ) diff --git a/services/real-time/test/acceptance/js/ReceiveUpdateTests.js b/services/real-time/test/acceptance/js/ReceiveUpdateTests.js index 92d29b014f..b8cd9858b6 100644 --- a/services/real-time/test/acceptance/js/ReceiveUpdateTests.js +++ b/services/real-time/test/acceptance/js/ReceiveUpdateTests.js @@ -31,11 +31,11 @@ describe('receiveUpdate', function () { return async.series( [ - (cb) => { + cb => { return FixturesManager.setUpProject( { privilegeLevel: 'owner', - project: { name: 'Test Project' } + project: { name: 'Test Project' }, }, (error, { user_id, project_id }) => { this.user_id = user_id @@ -45,7 +45,7 @@ describe('receiveUpdate', function () { ) }, - (cb) => { + cb => { return FixturesManager.setUpDoc( this.project_id, { lines: this.lines, version: this.version, ops: this.ops }, @@ -56,49 +56,49 @@ describe('receiveUpdate', function () { ) }, - (cb) => { + cb => { this.clientA = RealTimeClient.connect() return this.clientA.on('connectionAccepted', cb) }, - (cb) => { + cb => { this.clientB = RealTimeClient.connect() return this.clientB.on('connectionAccepted', cb) }, - (cb) => { + cb => { return this.clientA.emit( 'joinProject', { - project_id: this.project_id + project_id: this.project_id, }, cb ) }, - (cb) => { + cb => { return this.clientA.emit('joinDoc', this.doc_id, cb) }, - (cb) => { + cb => { return this.clientB.emit( 'joinProject', { - project_id: this.project_id + project_id: this.project_id, }, cb ) }, - (cb) => { + cb => { return this.clientB.emit('joinDoc', this.doc_id, cb) }, - (cb) => { + cb => { return FixturesManager.setUpProject( { privilegeLevel: 'owner', - project: { name: 'Test Project' } + project: { name: 'Test Project' }, }, ( error, @@ -111,7 +111,7 @@ describe('receiveUpdate', function () { ) }, - (cb) => { + cb => { return FixturesManager.setUpDoc( this.project_id_second, { lines: this.lines, version: this.version, ops: this.ops }, @@ -122,52 +122,52 @@ describe('receiveUpdate', function () { ) }, - (cb) => { + cb => { this.clientC = RealTimeClient.connect() return this.clientC.on('connectionAccepted', cb) }, - (cb) => { + cb => { return this.clientC.emit( 'joinProject', { - project_id: this.project_id_second + project_id: this.project_id_second, }, cb ) }, - (cb) => { + cb => { return this.clientC.emit('joinDoc', this.doc_id_second, cb) }, - (cb) => { + cb => { this.clientAUpdates = [] - this.clientA.on('otUpdateApplied', (update) => + this.clientA.on('otUpdateApplied', update => this.clientAUpdates.push(update) ) this.clientBUpdates = [] - this.clientB.on('otUpdateApplied', (update) => + this.clientB.on('otUpdateApplied', update => this.clientBUpdates.push(update) ) this.clientCUpdates = [] - this.clientC.on('otUpdateApplied', (update) => + this.clientC.on('otUpdateApplied', update => this.clientCUpdates.push(update) ) this.clientAErrors = [] - this.clientA.on('otUpdateError', (error) => + this.clientA.on('otUpdateError', error => this.clientAErrors.push(error) ) this.clientBErrors = [] - this.clientB.on('otUpdateError', (error) => + this.clientB.on('otUpdateError', error => this.clientBErrors.push(error) ) this.clientCErrors = [] - this.clientC.on('otUpdateError', (error) => + this.clientC.on('otUpdateError', error => this.clientCErrors.push(error) ) return cb() - } + }, ], done ) @@ -189,12 +189,12 @@ describe('receiveUpdate', function () { doc_id: this.doc_id, op: { meta: { - source: this.clientA.publicId + source: this.clientA.publicId, }, v: this.version, doc: this.doc_id, - op: [{ i: 'foo', p: 50 }] - } + op: [{ i: 'foo', p: 50 }], + }, } rclient.publish('applied-ops', JSON.stringify(this.update)) return setTimeout(done, 200) @@ -208,8 +208,8 @@ describe('receiveUpdate', function () { return this.clientAUpdates.should.deep.equal([ { v: this.version, - doc: this.doc_id - } + doc: this.doc_id, + }, ]) }) @@ -224,12 +224,12 @@ describe('receiveUpdate', function () { doc_id: this.doc_id_second, op: { meta: { - source: this.clientC.publicId + source: this.clientC.publicId, }, v: this.version, doc: this.doc_id_second, - op: [{ i: 'update from clientC', p: 50 }] - } + op: [{ i: 'update from clientC', p: 50 }], + }, } rclient.publish('applied-ops', JSON.stringify(this.update)) return setTimeout(done, 200) @@ -247,8 +247,8 @@ describe('receiveUpdate', function () { return this.clientCUpdates.should.deep.equal([ { v: this.version, - doc: this.doc_id_second - } + doc: this.doc_id_second, + }, ]) }) }) @@ -259,12 +259,12 @@ describe('receiveUpdate', function () { doc_id: this.doc_id, op: { meta: { - source: 'this-is-a-remote-client-id' + source: 'this-is-a-remote-client-id', }, v: this.version, doc: this.doc_id, - op: [{ i: 'foo', p: 50 }] - } + op: [{ i: 'foo', p: 50 }], + }, } rclient.publish('applied-ops', JSON.stringify(this.update)) return setTimeout(done, 200) @@ -289,7 +289,7 @@ describe('receiveUpdate', function () { 'applied-ops', JSON.stringify({ doc_id: this.doc_id, - error: (this.error = 'something went wrong') + error: (this.error = 'something went wrong'), }) ) return setTimeout(done, 200) @@ -320,7 +320,7 @@ describe('receiveUpdate', function () { 'applied-ops', JSON.stringify({ doc_id: this.doc_id_second, - error: (this.error = 'something went wrong') + error: (this.error = 'something went wrong'), }) ) return setTimeout(done, 200) diff --git a/services/real-time/test/acceptance/js/RouterTests.js b/services/real-time/test/acceptance/js/RouterTests.js index 729947281c..cc3862975d 100644 --- a/services/real-time/test/acceptance/js/RouterTests.js +++ b/services/real-time/test/acceptance/js/RouterTests.js @@ -22,17 +22,17 @@ describe('Router', function () { }) before(function (done) { - this.onUnhandled = (error) => done(error) + this.onUnhandled = error => done(error) process.on('unhandledRejection', this.onUnhandled) return async.series( [ - (cb) => { + cb => { return FixturesManager.setUpProject( { privilegeLevel: 'owner', project: { - name: 'Test Project' - } + name: 'Test Project', + }, }, (e, { project_id, user_id }) => { this.project_id = project_id @@ -42,20 +42,20 @@ describe('Router', function () { ) }, - (cb) => { + cb => { this.client = RealTimeClient.connect() return this.client.on('connectionAccepted', cb) }, - (cb) => { + cb => { this.client = RealTimeClient.connect() return this.client.on('connectionAccepted', cb) }, - (cb) => { + cb => { this.client.emit('joinProject', { project_id: this.project_id }) return setTimeout(cb, 100) - } + }, ], done ) @@ -72,17 +72,17 @@ describe('Router', function () { }) before(function (done) { - this.onUnhandled = (error) => done(error) + this.onUnhandled = error => done(error) process.on('unhandledRejection', this.onUnhandled) return async.series( [ - (cb) => { + cb => { return FixturesManager.setUpProject( { privilegeLevel: 'owner', project: { - name: 'Test Project' - } + name: 'Test Project', + }, }, (e, { project_id, user_id }) => { this.project_id = project_id @@ -92,22 +92,22 @@ describe('Router', function () { ) }, - (cb) => { + cb => { this.client = RealTimeClient.connect() return this.client.on('connectionAccepted', cb) }, - (cb) => { + cb => { this.client = RealTimeClient.connect() return this.client.on('connectionAccepted', cb) }, - (cb) => { - return this.client.emit('joinProject', 1, 2, 3, 4, 5, (error) => { + cb => { + return this.client.emit('joinProject', 1, 2, 3, 4, 5, error => { this.error = error return cb() }) - } + }, ], done ) diff --git a/services/real-time/test/acceptance/js/SessionSocketsTests.js b/services/real-time/test/acceptance/js/SessionSocketsTests.js index 44835142ea..ae636fdd96 100644 --- a/services/real-time/test/acceptance/js/SessionSocketsTests.js +++ b/services/real-time/test/acceptance/js/SessionSocketsTests.js @@ -28,7 +28,7 @@ describe('SessionSockets', function () { }) return it('should return a lookup error', function (done) { - return this.checkSocket((error) => { + return this.checkSocket(error => { expect(error).to.exist expect(error.message).to.equal('invalid session') return done() @@ -42,7 +42,7 @@ describe('SessionSockets', function () { }) return it('should return a lookup error', function (done) { - return this.checkSocket((error) => { + return this.checkSocket(error => { expect(error).to.exist expect(error.message).to.equal('invalid session') return done() @@ -52,7 +52,7 @@ describe('SessionSockets', function () { describe('with an invalid cookie', function () { before(function (done) { - RealTimeClient.setSession({}, (error) => { + RealTimeClient.setSession({}, error => { if (error) { return done(error) } @@ -65,7 +65,7 @@ describe('SessionSockets', function () { }) return it('should return a lookup error', function (done) { - return this.checkSocket((error) => { + return this.checkSocket(error => { expect(error).to.exist expect(error.message).to.equal('invalid session') return done() @@ -79,7 +79,7 @@ describe('SessionSockets', function () { }) return it('should return a lookup error', function (done) { - return this.checkSocket((error) => { + return this.checkSocket(error => { expect(error).to.exist expect(error.message).to.equal('invalid session') return done() @@ -94,7 +94,7 @@ describe('SessionSockets', function () { }) return it('should not return an error', function (done) { - return this.checkSocket((error) => { + return this.checkSocket(error => { expect(error).to.not.exist return done() }) diff --git a/services/real-time/test/acceptance/js/SessionTests.js b/services/real-time/test/acceptance/js/SessionTests.js index 858a2dd48a..21d691bea3 100644 --- a/services/real-time/test/acceptance/js/SessionTests.js +++ b/services/real-time/test/acceptance/js/SessionTests.js @@ -21,9 +21,9 @@ describe('Session', function () { this.user_id = 'mock-user-id' RealTimeClient.setSession( { - user: { _id: this.user_id } + user: { _id: this.user_id }, }, - (error) => { + error => { if (error != null) { throw error } diff --git a/services/real-time/test/acceptance/js/helpers/FixturesManager.js b/services/real-time/test/acceptance/js/helpers/FixturesManager.js index b3b7aaa78d..48fdc16ada 100644 --- a/services/real-time/test/acceptance/js/helpers/FixturesManager.js +++ b/services/real-time/test/acceptance/js/helpers/FixturesManager.js @@ -32,13 +32,8 @@ module.exports = FixturesManager = { if (!options.project) { options.project = { name: 'Test Project' } } - const { - project_id, - user_id, - privilegeLevel, - project, - publicAccess - } = options + const { project_id, user_id, privilegeLevel, project, publicAccess } = + options const privileges = {} privileges[user_id] = privilegeLevel @@ -47,7 +42,7 @@ module.exports = FixturesManager = { } MockWebServer.createMockProject(project_id, privileges, project) - return MockWebServer.run((error) => { + return MockWebServer.run(error => { if (error != null) { throw error } @@ -56,10 +51,10 @@ module.exports = FixturesManager = { user: { _id: user_id, first_name: 'Joe', - last_name: 'Bloggs' - } + last_name: 'Bloggs', + }, }, - (error) => { + error => { if (error != null) { throw error } @@ -67,7 +62,7 @@ module.exports = FixturesManager = { project_id, user_id, privilegeLevel, - project + project, }) } ) @@ -99,9 +94,9 @@ module.exports = FixturesManager = { lines, version, ops, - ranges + ranges, }) - return MockDocUpdaterServer.run((error) => { + return MockDocUpdaterServer.run(error => { if (error != null) { throw error } @@ -131,5 +126,5 @@ module.exports = FixturesManager = { .update(Math.random().toString()) .digest('hex') .slice(0, 24) - } + }, } diff --git a/services/real-time/test/acceptance/js/helpers/MockDocUpdaterServer.js b/services/real-time/test/acceptance/js/helpers/MockDocUpdaterServer.js index 26cc4722a0..519da94745 100644 --- a/services/real-time/test/acceptance/js/helpers/MockDocUpdaterServer.js +++ b/services/real-time/test/acceptance/js/helpers/MockDocUpdaterServer.js @@ -53,7 +53,7 @@ module.exports = MockDocUpdaterServer = { deleteProjectRequest(req, res, next) { const { project_id } = req.params - return MockDocUpdaterServer.deleteProject(project_id, (error) => { + return MockDocUpdaterServer.deleteProject(project_id, error => { if (error != null) { return next(error) } @@ -79,15 +79,15 @@ module.exports = MockDocUpdaterServer = { MockDocUpdaterServer.deleteProjectRequest ) return app - .listen(3003, (error) => { + .listen(3003, error => { MockDocUpdaterServer.running = true return callback(error) }) - .on('error', (error) => { + .on('error', error => { console.error('error starting MockDocUpdaterServer:', error.message) return process.exit(1) }) - } + }, } sinon.spy(MockDocUpdaterServer, 'getDocument') diff --git a/services/real-time/test/acceptance/js/helpers/MockWebServer.js b/services/real-time/test/acceptance/js/helpers/MockWebServer.js index 160996c6b9..c84bfd14b6 100644 --- a/services/real-time/test/acceptance/js/helpers/MockWebServer.js +++ b/services/real-time/test/acceptance/js/helpers/MockWebServer.js @@ -57,7 +57,7 @@ module.exports = MockWebServer = { } return res.json({ project, - privilegeLevel + privilegeLevel, }) } ) @@ -75,15 +75,15 @@ module.exports = MockWebServer = { const app = express() app.post('/project/:project_id/join', MockWebServer.joinProjectRequest) return app - .listen(3000, (error) => { + .listen(3000, error => { MockWebServer.running = true return callback(error) }) - .on('error', (error) => { + .on('error', error => { console.error('error starting MockWebServer:', error.message) return process.exit(1) }) - } + }, } sinon.spy(MockWebServer, 'joinProject') diff --git a/services/real-time/test/acceptance/js/helpers/RealTimeClient.js b/services/real-time/test/acceptance/js/helpers/RealTimeClient.js index d5e8b79343..00f00b5ee9 100644 --- a/services/real-time/test/acceptance/js/helpers/RealTimeClient.js +++ b/services/real-time/test/acceptance/js/helpers/RealTimeClient.js @@ -45,19 +45,15 @@ module.exports = Client = { } const sessionId = uid(24) session.cookie = {} - return rclient.set( - 'sess:' + sessionId, - JSON.stringify(session), - (error) => { - if (error != null) { - return callback(error) - } - const secret = Settings.security.sessionSecret - const cookieKey = 's:' + signature.sign(sessionId, secret) - Client.cookie = `${Settings.cookieName}=${cookieKey}` - return callback() + return rclient.set('sess:' + sessionId, JSON.stringify(session), error => { + if (error != null) { + return callback(error) } - ) + const secret = Settings.security.sessionSecret + const cookieKey = 's:' + signature.sign(sessionId, secret) + Client.cookie = `${Settings.cookieName}=${cookieKey}` + return callback() + }) }, unsetSession(callback) { @@ -70,7 +66,7 @@ module.exports = Client = { connect(cookie) { const client = io.connect('http://localhost:3026', { - 'force new connection': true + 'force new connection': true, }) client.on( 'connectionAccepted', @@ -86,7 +82,7 @@ module.exports = Client = { return request.get( { url: 'http://localhost:3026/clients', - json: true + json: true, }, (error, response, data) => callback(error, data) ) @@ -99,7 +95,7 @@ module.exports = Client = { return request.get( { url: `http://localhost:3026/clients/${client_id}`, - json: true + json: true, }, (error, response, data) => callback(error, data) ) @@ -111,8 +107,8 @@ module.exports = Client = { url: `http://localhost:3026/client/${client_id}/disconnect`, auth: { user: Settings.internal.realTime.user, - pass: Settings.internal.realTime.pass - } + pass: Settings.internal.realTime.pass, + }, }, (error, response, data) => callback(error, data) ) @@ -127,5 +123,5 @@ module.exports = Client = { callback ) ) - } + }, } diff --git a/services/real-time/test/acceptance/js/helpers/RealtimeServer.js b/services/real-time/test/acceptance/js/helpers/RealtimeServer.js index ef1a85a2a5..e964765b7a 100644 --- a/services/real-time/test/acceptance/js/helpers/RealtimeServer.js +++ b/services/real-time/test/acceptance/js/helpers/RealtimeServer.js @@ -34,10 +34,10 @@ module.exports = { return app.listen( __guard__( Settings.internal != null ? Settings.internal.realtime : undefined, - (x) => x.port + x => x.port ), 'localhost', - (error) => { + error => { if (error != null) { throw error } @@ -54,7 +54,7 @@ module.exports = { } ) } - } + }, } function __guard__(value, transform) { diff --git a/services/real-time/test/acceptance/libs/XMLHttpRequest.js b/services/real-time/test/acceptance/libs/XMLHttpRequest.js index 0222bc906b..b0436718e3 100644 --- a/services/real-time/test/acceptance/libs/XMLHttpRequest.js +++ b/services/real-time/test/acceptance/libs/XMLHttpRequest.js @@ -12,36 +12,36 @@ */ const { URL } = require('url') -var spawn = require('child_process').spawn -var fs = require('fs') +const spawn = require('child_process').spawn +const fs = require('fs') exports.XMLHttpRequest = function () { /** * Private variables */ - var self = this - var http = require('http') - var https = require('https') + const self = this + const http = require('http') + const https = require('https') // Holds http.js objects - var request - var response + let request + let response // Request settings - var settings = {} + let settings = {} // Set some default headers - var defaultHeaders = { + const defaultHeaders = { 'User-Agent': 'node-XMLHttpRequest', - Accept: '*/*' + Accept: '*/*', } - var headers = defaultHeaders + let headers = defaultHeaders // These headers are not user setable. // The following are allowed but banned in the spec: // * user-agent - var forbiddenRequestHeaders = [ + const forbiddenRequestHeaders = [ 'accept-charset', 'accept-encoding', 'access-control-request-headers', @@ -61,19 +61,19 @@ exports.XMLHttpRequest = function () { 'trailer', 'transfer-encoding', 'upgrade', - 'via' + 'via', ] // These request methods are not allowed - var forbiddenRequestMethods = ['TRACE', 'TRACK', 'CONNECT'] + const forbiddenRequestMethods = ['TRACE', 'TRACK', 'CONNECT'] // Send flag - var sendFlag = false + let sendFlag = false // Error flag, used when errors occur or abort is called - var errorFlag = false + let errorFlag = false // Event listeners - var listeners = {} + const listeners = {} /** * Constants @@ -111,7 +111,7 @@ exports.XMLHttpRequest = function () { * @param string header Header to validate * @return boolean False if not allowed, otherwise true */ - var isAllowedHttpHeader = function (header) { + const isAllowedHttpHeader = function (header) { return ( header && forbiddenRequestHeaders.indexOf(header.toLowerCase()) === -1 ) @@ -123,7 +123,7 @@ exports.XMLHttpRequest = function () { * @param string method Request method to validate * @return boolean False if not allowed, otherwise true */ - var isAllowedHttpMethod = function (method) { + const isAllowedHttpMethod = function (method) { return method && forbiddenRequestMethods.indexOf(method) === -1 } @@ -154,7 +154,7 @@ exports.XMLHttpRequest = function () { url: url.toString(), async: typeof async !== 'boolean' ? true : async, user: user || null, - password: password || null + password: password || null, } setState(this.OPENED) @@ -210,9 +210,9 @@ exports.XMLHttpRequest = function () { if (this.readyState < this.HEADERS_RECEIVED || errorFlag) { return '' } - var result = '' + let result = '' - for (var i in response.headers) { + for (const i in response.headers) { // Cookie headers are excluded if (i !== 'set-cookie' && i !== 'set-cookie2') { result += i + ': ' + response.headers[i] + '\r\n' @@ -252,10 +252,10 @@ exports.XMLHttpRequest = function () { throw new Error('INVALID_STATE_ERR: send has already been called') } - var host - var ssl = false - var local = false - var url = new URL(settings.url) + let host + let ssl = false + let local = false + const url = new URL(settings.url) // Determine the server switch (url.protocol) { @@ -311,9 +311,9 @@ exports.XMLHttpRequest = function () { // Default to port 80. If accessing localhost on another port be sure // to use http://localhost:port/path - var port = url.port || (ssl ? 443 : 80) + const port = url.port || (ssl ? 443 : 80) // Add query string if one is used - var uri = url.pathname + (url.search ? url.search : '') + const uri = url.pathname + (url.search ? url.search : '') // Set the Host header or the server may reject the request headers.Host = host @@ -326,7 +326,7 @@ exports.XMLHttpRequest = function () { if (typeof settings.password === 'undefined') { settings.password = '' } - var authBuf = Buffer.from(settings.user + ':' + settings.password) + const authBuf = Buffer.from(settings.user + ':' + settings.password) headers.Authorization = 'Basic ' + authBuf.toString('base64') } @@ -345,12 +345,12 @@ exports.XMLHttpRequest = function () { headers['Content-Length'] = 0 } - var options = { + const options = { host: host, port: port, path: uri, method: settings.method, - headers: headers + headers: headers, } // Reset error flag @@ -359,7 +359,7 @@ exports.XMLHttpRequest = function () { // Handle async requests if (settings.async) { // Use the proper protocol - var doRequest = ssl ? https.request : http.request + const doRequest = ssl ? https.request : http.request // Request is being sent, set send flag sendFlag = true @@ -368,14 +368,14 @@ exports.XMLHttpRequest = function () { self.dispatchEvent('readystatechange') // Create the request - request = doRequest(options, (resp) => { + request = doRequest(options, resp => { response = resp response.setEncoding('utf8') setState(self.HEADERS_RECEIVED) self.status = response.statusCode - response.on('data', (chunk) => { + response.on('data', chunk => { // Make sure there's some data if (chunk) { self.responseText += chunk @@ -394,10 +394,10 @@ exports.XMLHttpRequest = function () { } }) - response.on('error', (error) => { + response.on('error', error => { self.handleError(error) }) - }).on('error', (error) => { + }).on('error', error => { self.handleError(error) }) @@ -412,10 +412,10 @@ exports.XMLHttpRequest = function () { } else { // Synchronous // Create a temporary file for communication with the other Node process - var syncFile = '.node-xmlhttprequest-sync-' + process.pid + const syncFile = '.node-xmlhttprequest-sync-' + process.pid fs.writeFileSync(syncFile, '', 'utf8') // The async request the other Node process executes - var execString = + const execString = "var http = require('http'), https = require('https'), fs = require('fs');" + 'var doRequest = http' + (ssl ? 's' : '') + @@ -457,7 +457,7 @@ exports.XMLHttpRequest = function () { fs.unlinkSync(syncFile) if (self.responseText.match(/^NODE-XMLHTTPREQUEST-ERROR:/)) { // If the file returned an error, handle it - var errorObj = self.responseText.replace( + const errorObj = self.responseText.replace( /^NODE-XMLHTTPREQUEST-ERROR:/, '' ) @@ -532,7 +532,7 @@ exports.XMLHttpRequest = function () { this.removeEventListener = function (event, callback) { if (event in listeners) { // Filter will return a new array with the callback removed - listeners[event] = listeners[event].filter((ev) => { + listeners[event] = listeners[event].filter(ev => { return ev !== callback }) } @@ -546,7 +546,7 @@ exports.XMLHttpRequest = function () { self['on' + event]() } if (event in listeners) { - for (var i = 0, len = listeners[event].length; i < len; i++) { + for (let i = 0, len = listeners[event].length; i < len; i++) { listeners[event][i].call(self) } } diff --git a/services/real-time/test/setup.js b/services/real-time/test/setup.js index 90f4363c52..19520444e8 100644 --- a/services/real-time/test/setup.js +++ b/services/real-time/test/setup.js @@ -14,16 +14,16 @@ const stubs = { info: sandbox.stub(), warn: sandbox.stub(), err: sandbox.stub(), - error: sandbox.stub() - } + error: sandbox.stub(), + }, } // SandboxedModule configuration SandboxedModule.configure({ requires: { - 'logger-sharelatex': stubs.logger + 'logger-sharelatex': stubs.logger, }, - globals: { Buffer, JSON, console, process } + globals: { Buffer, JSON, console, process }, }) // Mocha hooks @@ -34,5 +34,5 @@ exports.mochaHooks = { afterEach() { sandbox.reset() - } + }, } diff --git a/services/real-time/test/unit/js/AuthorizationManagerTests.js b/services/real-time/test/unit/js/AuthorizationManagerTests.js index d8a3f7ae66..422ce8d15d 100644 --- a/services/real-time/test/unit/js/AuthorizationManagerTests.js +++ b/services/real-time/test/unit/js/AuthorizationManagerTests.js @@ -20,7 +20,7 @@ describe('AuthorizationManager', function () { this.client = { ol_context: {} } return (this.AuthorizationManager = SandboxedModule.require(modulePath, { - requires: {} + requires: {}, })) }) @@ -29,7 +29,7 @@ describe('AuthorizationManager', function () { this.client.ol_context.privilege_level = 'readOnly' return this.AuthorizationManager.assertClientCanViewProject( this.client, - (error) => { + error => { expect(error).to.be.null return done() } @@ -40,7 +40,7 @@ describe('AuthorizationManager', function () { this.client.ol_context.privilege_level = 'readAndWrite' return this.AuthorizationManager.assertClientCanViewProject( this.client, - (error) => { + error => { expect(error).to.be.null return done() } @@ -51,7 +51,7 @@ describe('AuthorizationManager', function () { this.client.ol_context.privilege_level = 'owner' return this.AuthorizationManager.assertClientCanViewProject( this.client, - (error) => { + error => { expect(error).to.be.null return done() } @@ -62,7 +62,7 @@ describe('AuthorizationManager', function () { this.client.ol_context.privilege_level = 'unknown' return this.AuthorizationManager.assertClientCanViewProject( this.client, - (error) => { + error => { error.message.should.equal('not authorized') return done() } @@ -75,7 +75,7 @@ describe('AuthorizationManager', function () { this.client.ol_context.privilege_level = 'readOnly' return this.AuthorizationManager.assertClientCanEditProject( this.client, - (error) => { + error => { error.message.should.equal('not authorized') return done() } @@ -86,7 +86,7 @@ describe('AuthorizationManager', function () { this.client.ol_context.privilege_level = 'readAndWrite' return this.AuthorizationManager.assertClientCanEditProject( this.client, - (error) => { + error => { expect(error).to.be.null return done() } @@ -97,7 +97,7 @@ describe('AuthorizationManager', function () { this.client.ol_context.privilege_level = 'owner' return this.AuthorizationManager.assertClientCanEditProject( this.client, - (error) => { + error => { expect(error).to.be.null return done() } @@ -108,7 +108,7 @@ describe('AuthorizationManager', function () { this.client.ol_context.privilege_level = 'unknown' return this.AuthorizationManager.assertClientCanEditProject( this.client, - (error) => { + error => { error.message.should.equal('not authorized') return done() } @@ -134,7 +134,7 @@ describe('AuthorizationManager', function () { return this.AuthorizationManager.assertClientCanViewProjectAndDoc( this.client, this.doc_id, - (err) => err.message.should.equal('not authorized') + err => err.message.should.equal('not authorized') ) }) @@ -151,7 +151,7 @@ describe('AuthorizationManager', function () { return this.AuthorizationManager.assertClientCanViewProjectAndDoc( this.client, this.doc_id, - (err) => err.message.should.equal('not authorized') + err => err.message.should.equal('not authorized') ) }) }) @@ -167,7 +167,7 @@ describe('AuthorizationManager', function () { return this.AuthorizationManager.assertClientCanViewProjectAndDoc( this.client, this.doc_id, - (err) => err.message.should.equal('not authorized') + err => err.message.should.equal('not authorized') ) }) }) @@ -210,7 +210,7 @@ describe('AuthorizationManager', function () { return this.AuthorizationManager.assertClientCanViewProjectAndDoc( this.client, this.doc_id, - (err) => err.message.should.equal('not authorized') + err => err.message.should.equal('not authorized') ) }) }) @@ -233,7 +233,7 @@ describe('AuthorizationManager', function () { return this.AuthorizationManager.assertClientCanEditProjectAndDoc( this.client, this.doc_id, - (err) => err.message.should.equal('not authorized') + err => err.message.should.equal('not authorized') ) }) @@ -250,7 +250,7 @@ describe('AuthorizationManager', function () { return this.AuthorizationManager.assertClientCanEditProjectAndDoc( this.client, this.doc_id, - (err) => err.message.should.equal('not authorized') + err => err.message.should.equal('not authorized') ) }) }) @@ -266,7 +266,7 @@ describe('AuthorizationManager', function () { return this.AuthorizationManager.assertClientCanEditProjectAndDoc( this.client, this.doc_id, - (err) => err.message.should.equal('not authorized') + err => err.message.should.equal('not authorized') ) }) }) @@ -309,7 +309,7 @@ describe('AuthorizationManager', function () { return this.AuthorizationManager.assertClientCanEditProjectAndDoc( this.client, this.doc_id, - (err) => err.message.should.equal('not authorized') + err => err.message.should.equal('not authorized') ) }) }) diff --git a/services/real-time/test/unit/js/ChannelManagerTests.js b/services/real-time/test/unit/js/ChannelManagerTests.js index a7d857dada..2e51c584f2 100644 --- a/services/real-time/test/unit/js/ChannelManagerTests.js +++ b/services/real-time/test/unit/js/ChannelManagerTests.js @@ -23,9 +23,9 @@ describe('ChannelManager', function () { '@overleaf/settings': (this.settings = {}), '@overleaf/metrics': (this.metrics = { inc: sinon.stub(), - summary: sinon.stub() - }) - } + summary: sinon.stub(), + }), + }, })) }) @@ -83,7 +83,7 @@ describe('ChannelManager', function () { '1234567890abcdef' ) p.then(() => done(new Error('should not subscribe but fail'))).catch( - (err) => { + err => { err.message.should.equal('failed to subscribe to channel') err.cause.message.should.equal('some redis error') this.ChannelManager.getClientMapEntry(this.rclient) diff --git a/services/real-time/test/unit/js/ConnectedUsersManagerTests.js b/services/real-time/test/unit/js/ConnectedUsersManagerTests.js index 8525b94b02..51abee1bc7 100644 --- a/services/real-time/test/unit/js/ConnectedUsersManagerTests.js +++ b/services/real-time/test/unit/js/ConnectedUsersManagerTests.js @@ -31,10 +31,10 @@ describe('ConnectedUsersManager', function () { }, connectedUser({ project_id, client_id }) { return `connected_user:${project_id}:${client_id}` - } - } - } - } + }, + }, + }, + }, } this.rClient = { auth() {}, @@ -50,7 +50,7 @@ describe('ConnectedUsersManager', function () { exec: sinon.stub(), multi: () => { return this.rClient - } + }, } tk.freeze(new Date()) @@ -60,9 +60,9 @@ describe('ConnectedUsersManager', function () { '@overleaf/redis-wrapper': { createClient: () => { return this.rClient - } - } - } + }, + }, + }, }) this.client_id = '32132132' this.project_id = 'dskjh2u21321' @@ -70,12 +70,12 @@ describe('ConnectedUsersManager', function () { _id: 'user-id-123', first_name: 'Joe', last_name: 'Bloggs', - email: 'joe@example.com' + email: 'joe@example.com', } return (this.cursorData = { row: 12, column: 9, - doc_id: '53c3b8c85fee64000023dc6e' + doc_id: '53c3b8c85fee64000023dc6e', }) }) @@ -94,7 +94,7 @@ describe('ConnectedUsersManager', function () { this.client_id, this.user, null, - (err) => { + err => { this.rClient.hset .calledWith( `connected_user:${this.project_id}:${this.client_id}`, @@ -113,7 +113,7 @@ describe('ConnectedUsersManager', function () { this.client_id, this.user, null, - (err) => { + err => { this.rClient.hset .calledWith( `connected_user:${this.project_id}:${this.client_id}`, @@ -132,7 +132,7 @@ describe('ConnectedUsersManager', function () { this.client_id, this.user, null, - (err) => { + err => { this.rClient.hset .calledWith( `connected_user:${this.project_id}:${this.client_id}`, @@ -151,7 +151,7 @@ describe('ConnectedUsersManager', function () { this.client_id, this.user, null, - (err) => { + err => { this.rClient.hset .calledWith( `connected_user:${this.project_id}:${this.client_id}`, @@ -170,7 +170,7 @@ describe('ConnectedUsersManager', function () { this.client_id, this.user, null, - (err) => { + err => { this.rClient.hset .calledWith( `connected_user:${this.project_id}:${this.client_id}`, @@ -189,7 +189,7 @@ describe('ConnectedUsersManager', function () { this.client_id, this.user, null, - (err) => { + err => { this.rClient.sadd .calledWith(`clients_in_project:${this.project_id}`, this.client_id) .should.equal(true) @@ -204,7 +204,7 @@ describe('ConnectedUsersManager', function () { this.client_id, this.user, null, - (err) => { + err => { this.rClient.expire .calledWith( `clients_in_project:${this.project_id}`, @@ -222,7 +222,7 @@ describe('ConnectedUsersManager', function () { this.client_id, this.user, null, - (err) => { + err => { this.rClient.expire .calledWith( `connected_user:${this.project_id}:${this.client_id}`, @@ -240,7 +240,7 @@ describe('ConnectedUsersManager', function () { this.client_id, this.user, this.cursorData, - (err) => { + err => { this.rClient.hset .calledWith( `connected_user:${this.project_id}:${this.client_id}`, @@ -263,7 +263,7 @@ describe('ConnectedUsersManager', function () { return this.ConnectedUsersManager.markUserAsDisconnected( this.project_id, this.client_id, - (err) => { + err => { this.rClient.srem .calledWith(`clients_in_project:${this.project_id}`, this.client_id) .should.equal(true) @@ -276,7 +276,7 @@ describe('ConnectedUsersManager', function () { return this.ConnectedUsersManager.markUserAsDisconnected( this.project_id, this.client_id, - (err) => { + err => { this.rClient.del .calledWith(`connected_user:${this.project_id}:${this.client_id}`) .should.equal(true) @@ -289,7 +289,7 @@ describe('ConnectedUsersManager', function () { return this.ConnectedUsersManager.markUserAsDisconnected( this.project_id, this.client_id, - (err) => { + err => { this.rClient.expire .calledWith( `clients_in_project:${this.project_id}`, @@ -309,7 +309,7 @@ describe('ConnectedUsersManager', function () { connected_at: new Date(), user_id: this.user._id, last_updated_at: `${Date.now()}`, - cursorData + cursorData, }) return this.ConnectedUsersManager._getConnectedUser( this.project_id, @@ -359,28 +359,28 @@ describe('ConnectedUsersManager', function () { .callsArgWith(2, null, { connected: true, client_age: 2, - client_id: this.users[0] + client_id: this.users[0], }) this.ConnectedUsersManager._getConnectedUser .withArgs(this.project_id, this.users[1]) .callsArgWith(2, null, { connected: false, client_age: 1, - client_id: this.users[1] + client_id: this.users[1], }) this.ConnectedUsersManager._getConnectedUser .withArgs(this.project_id, this.users[2]) .callsArgWith(2, null, { connected: true, client_age: 3, - client_id: this.users[2] + client_id: this.users[2], }) return this.ConnectedUsersManager._getConnectedUser .withArgs(this.project_id, this.users[3]) .callsArgWith(2, null, { connected: true, client_age: 11, - client_id: this.users[3] + client_id: this.users[3], }) }) // connected but old @@ -392,12 +392,12 @@ describe('ConnectedUsersManager', function () { users[0].should.deep.equal({ client_id: this.users[0], client_age: 2, - connected: true + connected: true, }) users[1].should.deep.equal({ client_id: this.users[2], client_age: 3, - connected: true + connected: true, }) return done() } diff --git a/services/real-time/test/unit/js/DocumentUpdaterControllerTests.js b/services/real-time/test/unit/js/DocumentUpdaterControllerTests.js index 333fe53499..20fa5bfe18 100644 --- a/services/real-time/test/unit/js/DocumentUpdaterControllerTests.js +++ b/services/real-time/test/unit/js/DocumentUpdaterControllerTests.js @@ -34,34 +34,34 @@ describe('DocumentUpdaterController', function () { key_schema: { pendingUpdates({ doc_id }) { return `PendingUpdates:${doc_id}` - } - } + }, + }, }, - pubsub: null - } + pubsub: null, + }, }), './RedisClientManager': { createClientList: () => { this.redis = { - createClient: (name) => { + createClient: name => { let rclientStub this.rclient.push((rclientStub = { name })) return rclientStub - } + }, } - } + }, }, './SafeJsonParse': (this.SafeJsonParse = { - parse: (data, cb) => cb(null, JSON.parse(data)) + parse: (data, cb) => cb(null, JSON.parse(data)), }), './EventLogger': (this.EventLogger = { checkEventOrder: sinon.stub() }), './HealthCheckManager': { check: sinon.stub() }, '@overleaf/metrics': (this.metrics = { inc: sinon.stub() }), './RoomManager': (this.RoomManager = { - eventSource: sinon.stub().returns(this.RoomEvents) + eventSource: sinon.stub().returns(this.RoomEvents), }), - './ChannelManager': (this.ChannelManager = {}) - } + './ChannelManager': (this.ChannelManager = {}), + }, }) }) @@ -70,7 +70,7 @@ describe('DocumentUpdaterController', function () { this.rclient.length = 0 // clear any existing clients this.EditorUpdatesController.rclientList = [ this.redis.createClient('first'), - this.redis.createClient('second') + this.redis.createClient('second'), ] this.rclient[0].subscribe = sinon.stub() this.rclient[0].on = sinon.stub() @@ -115,9 +115,10 @@ describe('DocumentUpdaterController', function () { beforeEach(function () { this.message = { doc_id: this.doc_id, - op: { t: 'foo', p: 12 } + op: { t: 'foo', p: 12 }, } - this.EditorUpdatesController._applyUpdateFromDocumentUpdater = sinon.stub() + this.EditorUpdatesController._applyUpdateFromDocumentUpdater = + sinon.stub() return this.EditorUpdatesController._processMessageFromDocumentUpdater( this.io, 'applied-ops', @@ -136,9 +137,10 @@ describe('DocumentUpdaterController', function () { beforeEach(function () { this.message = { doc_id: this.doc_id, - error: 'Something went wrong' + error: 'Something went wrong', } - this.EditorUpdatesController._processErrorFromDocumentUpdater = sinon.stub() + this.EditorUpdatesController._processErrorFromDocumentUpdater = + sinon.stub() return this.EditorUpdatesController._processMessageFromDocumentUpdater( this.io, 'applied-ops', @@ -162,7 +164,7 @@ describe('DocumentUpdaterController', function () { op: [{ t: 'foo', p: 12 }], meta: { source: this.sourceClient.publicId }, v: (this.version = 42), - doc: this.doc_id + doc: this.doc_id, } return (this.io.sockets = { clients: sinon @@ -170,8 +172,8 @@ describe('DocumentUpdaterController', function () { .returns([ this.sourceClient, ...Array.from(this.otherClients), - this.sourceClient - ]) + this.sourceClient, + ]), }) }) // include a duplicate client @@ -198,7 +200,7 @@ describe('DocumentUpdaterController', function () { }) return it('should send the full update to the other clients', function () { - return Array.from(this.otherClients).map((client) => + return Array.from(this.otherClients).map(client => client.emit .calledWith('otUpdateApplied', this.update) .should.equal(true) @@ -223,7 +225,7 @@ describe('DocumentUpdaterController', function () { }) return it("should not send anything to the other clients (they've already had the op)", function () { - return Array.from(this.otherClients).map((client) => + return Array.from(this.otherClients).map(client => client.emit.calledWith('otUpdateApplied').should.equal(false) ) }) @@ -247,7 +249,7 @@ describe('DocumentUpdaterController', function () { return it('should disconnect all clients in that document', function () { this.io.sockets.clients.calledWith(this.doc_id).should.equal(true) - return Array.from(this.clients).map((client) => + return Array.from(this.clients).map(client => client.disconnect.called.should.equal(true) ) }) diff --git a/services/real-time/test/unit/js/DocumentUpdaterManagerTests.js b/services/real-time/test/unit/js/DocumentUpdaterManagerTests.js index e2b6bdbe51..9d565b4a3b 100644 --- a/services/real-time/test/unit/js/DocumentUpdaterManagerTests.js +++ b/services/real-time/test/unit/js/DocumentUpdaterManagerTests.js @@ -30,12 +30,12 @@ describe('DocumentUpdaterManager', function () { key_schema: { pendingUpdates({ doc_id }) { return `PendingUpdates:${doc_id}` - } - } - } + }, + }, + }, }, maxUpdateSize: 7 * 1024 * 1024, - pendingUpdateListShardCount: 10 + pendingUpdateListShardCount: 10, } this.rclient = { auth() {} } @@ -48,9 +48,9 @@ describe('DocumentUpdaterManager', function () { summary: sinon.stub(), Timer: (Timer = class Timer { done() {} - }) - }) - } + }), + }), + }, })) }) // avoid modifying JSON object directly @@ -65,7 +65,7 @@ describe('DocumentUpdaterManager', function () { lines: this.lines, version: this.version, ops: (this.ops = ['mock-op-1', 'mock-op-2']), - ranges: (this.ranges = { mock: 'ranges' }) + ranges: (this.ranges = { mock: 'ranges' }), }) this.fromVersion = 2 this.request.get = sinon @@ -113,7 +113,7 @@ describe('DocumentUpdaterManager', function () { return this.callback.calledWith(this.error).should.equal(true) }) }) - ;[404, 422].forEach((statusCode) => + ;[404, 422].forEach(statusCode => describe(`when the document updater returns a ${statusCode} status code`, function () { beforeEach(function () { this.request.get = sinon @@ -133,7 +133,7 @@ describe('DocumentUpdaterManager', function () { .calledWith( sinon.match({ message: 'doc updater could not load requested ops', - info: { statusCode } + info: { statusCode }, }) ) .should.equal(true) @@ -164,8 +164,8 @@ describe('DocumentUpdaterManager', function () { message: 'doc updater returned a non-success status code', info: { action: 'getDocument', - statusCode: 500 - } + statusCode: 500, + }, }) ) .should.equal(true) @@ -240,8 +240,8 @@ describe('DocumentUpdaterManager', function () { message: 'doc updater returned a non-success status code', info: { action: 'flushProjectToMongoAndDelete', - statusCode: 500 - } + statusCode: 500, + }, }) ) .should.equal(true) @@ -254,7 +254,7 @@ describe('DocumentUpdaterManager', function () { this.change = { doc: '1234567890', op: [{ d: 'test', p: 345 }], - v: 789 + v: 789, } this.rclient.rpush = sinon.stub().yields() return (this.callback = sinon.stub()) @@ -345,7 +345,7 @@ describe('DocumentUpdaterManager', function () { describe('when the update is too large', function () { beforeEach(function () { this.change = { - op: { p: 12, t: 'update is too large'.repeat(1024 * 400) } + op: { p: 12, t: 'update is too large'.repeat(1024 * 400) }, } return this.DocumentUpdaterManager.queueChange( this.project_id, @@ -374,7 +374,7 @@ describe('DocumentUpdaterManager', function () { beforeEach(function () { this.change = { op: [{ d: 'test', p: 345 }], - version: 789 // not a valid key + version: 789, // not a valid key } return this.DocumentUpdaterManager.queueChange( this.project_id, diff --git a/services/real-time/test/unit/js/DrainManagerTests.js b/services/real-time/test/unit/js/DrainManagerTests.js index bf8de6a6e9..3075c647c7 100644 --- a/services/real-time/test/unit/js/DrainManagerTests.js +++ b/services/real-time/test/unit/js/DrainManagerTests.js @@ -19,8 +19,8 @@ describe('DrainManager', function () { this.DrainManager = SandboxedModule.require(modulePath, {}) return (this.io = { sockets: { - clients: sinon.stub() - } + clients: sinon.stub(), + }, }) }) @@ -30,7 +30,7 @@ describe('DrainManager', function () { for (let i = 0; i <= 5399; i++) { this.clients[i] = { id: i, - emit: sinon.stub() + emit: sinon.stub(), } } this.io.sockets.clients.returns(this.clients) @@ -50,7 +50,7 @@ describe('DrainManager', function () { for (let i = 0; i <= 9; i++) { this.clients[i] = { id: i, - emit: sinon.stub() + emit: sinon.stub(), } } return this.io.sockets.clients.returns(this.clients) @@ -62,7 +62,7 @@ describe('DrainManager', function () { }) it('should reconnect the first 3 clients', function () { - return [0, 1, 2].map((i) => + return [0, 1, 2].map(i => this.clients[i].emit .calledWith('reconnectGracefully') .should.equal(true) @@ -70,7 +70,7 @@ describe('DrainManager', function () { }) it('should not reconnect any more clients', function () { - return [3, 4, 5, 6, 7, 8, 9].map((i) => + return [3, 4, 5, 6, 7, 8, 9].map(i => this.clients[i].emit .calledWith('reconnectGracefully') .should.equal(false) @@ -83,7 +83,7 @@ describe('DrainManager', function () { }) it('should reconnect the next 3 clients', function () { - return [3, 4, 5].map((i) => + return [3, 4, 5].map(i => this.clients[i].emit .calledWith('reconnectGracefully') .should.equal(true) @@ -91,7 +91,7 @@ describe('DrainManager', function () { }) it('should not reconnect any more clients', function () { - return [6, 7, 8, 9].map((i) => + return [6, 7, 8, 9].map(i => this.clients[i].emit .calledWith('reconnectGracefully') .should.equal(false) @@ -99,7 +99,7 @@ describe('DrainManager', function () { }) it('should not reconnect the first 3 clients again', function () { - return [0, 1, 2].map((i) => + return [0, 1, 2].map(i => this.clients[i].emit.calledOnce.should.equal(true) ) }) @@ -110,7 +110,7 @@ describe('DrainManager', function () { }) it('should not reconnect the first 6 clients again', function () { - return [0, 1, 2, 3, 4, 5].map((i) => + return [0, 1, 2, 3, 4, 5].map(i => this.clients[i].emit.calledOnce.should.equal(true) ) }) diff --git a/services/real-time/test/unit/js/EventLoggerTests.js b/services/real-time/test/unit/js/EventLoggerTests.js index 340e17184e..f647741fcf 100644 --- a/services/real-time/test/unit/js/EventLoggerTests.js +++ b/services/real-time/test/unit/js/EventLoggerTests.js @@ -20,8 +20,8 @@ describe('EventLogger', function () { tk.freeze(new Date(this.start)) this.EventLogger = SandboxedModule.require(modulePath, { requires: { - '@overleaf/metrics': (this.metrics = { inc: sinon.stub() }) - } + '@overleaf/metrics': (this.metrics = { inc: sinon.stub() }), + }, }) this.channel = 'applied-ops' this.id_1 = 'random-hostname:abc-1' diff --git a/services/real-time/test/unit/js/RoomManagerTests.js b/services/real-time/test/unit/js/RoomManagerTests.js index b2cd2fe0e3..f33d2ecce2 100644 --- a/services/real-time/test/unit/js/RoomManagerTests.js +++ b/services/real-time/test/unit/js/RoomManagerTests.js @@ -24,8 +24,8 @@ describe('RoomManager', function () { this.RoomManager = SandboxedModule.require(modulePath, { requires: { '@overleaf/settings': (this.settings = {}), - '@overleaf/metrics': (this.metrics = { gauge: sinon.stub() }) - } + '@overleaf/metrics': (this.metrics = { gauge: sinon.stub() }), + }, }) this.RoomManager._clientsInRoom = sinon.stub() this.RoomManager._clientAlreadyInRoom = sinon.stub() @@ -41,7 +41,7 @@ describe('RoomManager', function () { }) beforeEach(function (done) { - this.onUnhandled = (error) => { + this.onUnhandled = error => { this.unhandledError = error return done(new Error(`unhandledRejection: ${error.message}`)) } @@ -71,7 +71,7 @@ describe('RoomManager', function () { .returns(0) this.client.join = sinon.stub() this.callback = sinon.stub() - this.RoomEvents.on('project-active', (id) => { + this.RoomEvents.on('project-active', id => { return setTimeout(() => { return this.RoomEvents.emit(`project-subscribed-${id}`) }, 100) @@ -79,7 +79,7 @@ describe('RoomManager', function () { return this.RoomManager.joinProject( this.client, this.project_id, - (err) => { + err => { this.callback(err) return done() } @@ -136,12 +136,12 @@ describe('RoomManager', function () { .returns(0) this.client.join = sinon.stub() this.callback = sinon.stub() - this.RoomEvents.on('doc-active', (id) => { + this.RoomEvents.on('doc-active', id => { return setTimeout(() => { return this.RoomEvents.emit(`doc-subscribed-${id}`) }, 100) }) - return this.RoomManager.joinDoc(this.client, this.doc_id, (err) => { + return this.RoomManager.joinDoc(this.client, this.doc_id, err => { this.callback(err) return done() }) @@ -301,12 +301,12 @@ describe('RoomManager', function () { .returns(true) .withArgs(this.client, this.project_id) .returns(true) - this.RoomEvents.on('project-active', (id) => { + this.RoomEvents.on('project-active', id => { return setTimeout(() => { return this.RoomEvents.emit(`project-subscribed-${id}`) }, 100) }) - this.RoomEvents.on('doc-active', (id) => { + this.RoomEvents.on('doc-active', id => { return setTimeout(() => { return this.RoomEvents.emit(`doc-subscribed-${id}`) }, 100) diff --git a/services/real-time/test/unit/js/SafeJsonParseTest.js b/services/real-time/test/unit/js/SafeJsonParseTest.js index bdbba00c93..dbdb4a41d2 100644 --- a/services/real-time/test/unit/js/SafeJsonParseTest.js +++ b/services/real-time/test/unit/js/SafeJsonParseTest.js @@ -20,9 +20,9 @@ describe('SafeJsonParse', function () { return (this.SafeJsonParse = SandboxedModule.require(modulePath, { requires: { '@overleaf/settings': (this.Settings = { - maxUpdateSize: 16 * 1024 - }) - } + maxUpdateSize: 16 * 1024, + }), + }, })) }) diff --git a/services/real-time/test/unit/js/SessionSocketsTests.js b/services/real-time/test/unit/js/SessionSocketsTests.js index f4ae34bf78..b6f6c87fb9 100644 --- a/services/real-time/test/unit/js/SessionSocketsTests.js +++ b/services/real-time/test/unit/js/SessionSocketsTests.js @@ -23,7 +23,7 @@ describe('SessionSockets', function () { this.id2 = Math.random().toString() const redisResponses = { error: [new Error('Redis: something went wrong'), null], - unknownId: [null, null] + unknownId: [null, null], } redisResponses[this.id1] = [null, { user: { _id: '123' } }] redisResponses[this.id2] = [null, { user: { _id: 'abc' } }] @@ -31,7 +31,7 @@ describe('SessionSockets', function () { this.sessionStore = { get: sinon .stub() - .callsFake((id, fn) => fn.apply(null, redisResponses[id])) + .callsFake((id, fn) => fn.apply(null, redisResponses[id])), } this.cookieParser = function (req, res, next) { req.signedCookies = req._signedCookies @@ -55,7 +55,7 @@ describe('SessionSockets', function () { }) it('should return a lookup error', function (done) { - return this.checkSocket(this.socket, (error) => { + return this.checkSocket(this.socket, error => { expect(error).to.exist expect(error.message).to.equal('could not look up session by key') return done() @@ -76,7 +76,7 @@ describe('SessionSockets', function () { }) it('should return a lookup error', function (done) { - return this.checkSocket(this.socket, (error) => { + return this.checkSocket(this.socket, error => { expect(error).to.exist expect(error.message).to.equal('could not look up session by key') return done() @@ -94,7 +94,7 @@ describe('SessionSockets', function () { describe('with a valid cookie and a failing session lookup', function () { before(function () { return (this.socket = { - handshake: { _signedCookies: { 'ol.sid': 'error' } } + handshake: { _signedCookies: { 'ol.sid': 'error' } }, }) }) @@ -106,7 +106,7 @@ describe('SessionSockets', function () { }) return it('should return a redis error', function (done) { - return this.checkSocket(this.socket, (error) => { + return this.checkSocket(this.socket, error => { expect(error).to.exist expect(error.message).to.equal('Redis: something went wrong') return done() @@ -117,7 +117,7 @@ describe('SessionSockets', function () { describe('with a valid cookie and no matching session', function () { before(function () { return (this.socket = { - handshake: { _signedCookies: { 'ol.sid': 'unknownId' } } + handshake: { _signedCookies: { 'ol.sid': 'unknownId' } }, }) }) @@ -129,7 +129,7 @@ describe('SessionSockets', function () { }) return it('should return a lookup error', function (done) { - return this.checkSocket(this.socket, (error) => { + return this.checkSocket(this.socket, error => { expect(error).to.exist expect(error.message).to.equal('could not look up session by key') return done() @@ -140,7 +140,7 @@ describe('SessionSockets', function () { describe('with a valid cookie and a matching session', function () { before(function () { return (this.socket = { - handshake: { _signedCookies: { 'ol.sid': this.id1 } } + handshake: { _signedCookies: { 'ol.sid': this.id1 } }, }) }) @@ -152,7 +152,7 @@ describe('SessionSockets', function () { }) it('should not return an error', function (done) { - return this.checkSocket(this.socket, (error) => { + return this.checkSocket(this.socket, error => { expect(error).to.not.exist return done() }) @@ -169,7 +169,7 @@ describe('SessionSockets', function () { return describe('with a different valid cookie and matching session', function () { before(function () { return (this.socket = { - handshake: { _signedCookies: { 'ol.sid': this.id2 } } + handshake: { _signedCookies: { 'ol.sid': this.id2 } }, }) }) @@ -181,7 +181,7 @@ describe('SessionSockets', function () { }) it('should not return an error', function (done) { - return this.checkSocket(this.socket, (error) => { + return this.checkSocket(this.socket, error => { expect(error).to.not.exist return done() }) diff --git a/services/real-time/test/unit/js/WebApiManagerTests.js b/services/real-time/test/unit/js/WebApiManagerTests.js index b1291b4205..922e15a8f8 100644 --- a/services/real-time/test/unit/js/WebApiManagerTests.js +++ b/services/real-time/test/unit/js/WebApiManagerTests.js @@ -28,11 +28,11 @@ describe('WebApiManager', function () { web: { url: 'http://web.example.com', user: 'username', - pass: 'password' - } - } - }) - } + pass: 'password', + }, + }, + }), + }, })) }) @@ -42,7 +42,7 @@ describe('WebApiManager', function () { this.response = { project: { name: 'Test project' }, privilegeLevel: 'owner', - isRestrictedUser: true + isRestrictedUser: true, } this.request.post = sinon .stub() @@ -59,16 +59,16 @@ describe('WebApiManager', function () { .calledWith({ url: `${this.settings.apis.web.url}/project/${this.project_id}/join`, qs: { - user_id: this.user_id + user_id: this.user_id, }, auth: { user: this.settings.apis.web.user, pass: this.settings.apis.web.pass, - sendImmediately: true + sendImmediately: true, }, json: true, jar: false, - headers: {} + headers: {}, }) .should.equal(true) }) @@ -101,7 +101,7 @@ describe('WebApiManager', function () { this.callback .calledWith( sinon.match({ - message: 'not authorized' + message: 'not authorized', }) ) .should.equal(true) @@ -125,7 +125,7 @@ describe('WebApiManager', function () { .calledWith( sinon.match({ message: 'project not found', - info: { code: 'ProjectNotFound' } + info: { code: 'ProjectNotFound' }, }) ) .should.equal(true) @@ -149,7 +149,7 @@ describe('WebApiManager', function () { .calledWith( sinon.match({ message: 'non-success status code from web', - info: { statusCode: 500 } + info: { statusCode: 500 }, }) ) .should.equal(true) @@ -172,7 +172,7 @@ describe('WebApiManager', function () { return this.callback .calledWith( sinon.match({ - message: 'no data returned from joinProject request' + message: 'no data returned from joinProject request', }) ) .should.equal(true) @@ -197,8 +197,8 @@ describe('WebApiManager', function () { sinon.match({ message: 'rate-limit hit when joining project', info: { - code: 'TooManyRequests' - } + code: 'TooManyRequests', + }, }) ) .should.equal(true) diff --git a/services/real-time/test/unit/js/WebsocketControllerTests.js b/services/real-time/test/unit/js/WebsocketControllerTests.js index e227a1ce82..8b7c60aed6 100644 --- a/services/real-time/test/unit/js/WebsocketControllerTests.js +++ b/services/real-time/test/unit/js/WebsocketControllerTests.js @@ -29,7 +29,7 @@ describe('WebsocketController', function () { last_name: 'Allen', email: 'james@example.com', signUpDate: new Date('2014-01-01'), - loginCount: 42 + loginCount: 42, } this.callback = sinon.stub() this.client = { @@ -39,7 +39,7 @@ describe('WebsocketController', function () { ol_context: {}, joinLeaveEpoch: 0, join: sinon.stub(), - leave: sinon.stub() + leave: sinon.stub(), } return (this.WebsocketController = SandboxedModule.require(modulePath, { requires: { @@ -50,10 +50,10 @@ describe('WebsocketController', function () { './WebsocketLoadBalancer': (this.WebsocketLoadBalancer = {}), '@overleaf/metrics': (this.metrics = { inc: sinon.stub(), - set: sinon.stub() + set: sinon.stub(), }), - './RoomManager': (this.RoomManager = {}) - } + './RoomManager': (this.RoomManager = {}), + }, })) }) @@ -68,8 +68,8 @@ describe('WebsocketController', function () { this.project = { name: 'Test Project', owner: { - _id: (this.owner_id = 'mock-owner-id-123') - } + _id: (this.owner_id = 'mock-owner-id-123'), + }, } this.privilegeLevel = 'owner' this.ConnectedUsersManager.updateUserPosition = sinon @@ -205,8 +205,8 @@ describe('WebsocketController', function () { this.project = { name: 'Test Project', owner: { - _id: (this.owner_id = 'mock-owner-id-123') - } + _id: (this.owner_id = 'mock-owner-id-123'), + }, } this.privilegeLevel = 'owner' this.ConnectedUsersManager.updateUserPosition = sinon @@ -264,7 +264,7 @@ describe('WebsocketController', function () { return it('should increment the editor.join-project.disconnected metric with a status', function () { return expect( this.metrics.inc.calledWith('editor.join-project.disconnected', 1, { - status: 'immediately' + status: 'immediately', }) ).to.equal(true) }) @@ -297,7 +297,7 @@ describe('WebsocketController', function () { return it('should increment the editor.join-project.disconnected metric with a status', function () { return expect( this.metrics.inc.calledWith('editor.join-project.disconnected', 1, { - status: 'after-web-api-call' + status: 'after-web-api-call', }) ).to.equal(true) }) @@ -317,13 +317,13 @@ describe('WebsocketController', function () { this.clientsInRoom = [] this.io = { sockets: { - clients: (room_id) => { + clients: room_id => { if (room_id !== this.project_id) { throw 'expected room_id to be project_id' } return this.clientsInRoom - } - } + }, + }, } this.client.ol_context.project_id = this.project_id this.client.ol_context.user_id = this.user_id @@ -396,13 +396,13 @@ describe('WebsocketController', function () { this.clientsInRoom = ['mock-remaining-client'] this.io = { sockets: { - clients: (room_id) => { + clients: room_id => { if (room_id !== this.project_id) { throw 'expected room_id to be project_id' } return this.clientsInRoom - } - } + }, + }, } return this.WebsocketController.leaveProject(this.io, this.client, done) }) @@ -729,7 +729,7 @@ describe('WebsocketController', function () { it('should increment the editor.join-doc.disconnected metric with a status', function () { return expect( this.metrics.inc.calledWith('editor.join-doc.disconnected', 1, { - status: 'immediately' + status: 'immediately', }) ).to.equal(true) }) @@ -775,7 +775,7 @@ describe('WebsocketController', function () { it('should increment the editor.join-doc.disconnected metric with a status', function () { expect( this.metrics.inc.calledWith('editor.join-doc.disconnected', 1, { - status: 'after-client-auth-check' + status: 'after-client-auth-check', }) ).to.equal(true) }) @@ -894,7 +894,7 @@ describe('WebsocketController', function () { it('should increment the editor.join-doc.disconnected metric with a status', function () { return expect( this.metrics.inc.calledWith('editor.join-doc.disconnected', 1, { - status: 'after-joining-room' + status: 'after-joining-room', }) ).to.equal(true) }) @@ -940,7 +940,7 @@ describe('WebsocketController', function () { return it('should increment the editor.join-doc.disconnected metric with a status', function () { return expect( this.metrics.inc.calledWith('editor.join-doc.disconnected', 1, { - status: 'after-doc-updater-call' + status: 'after-doc-updater-call', }) ).to.equal(true) }) @@ -1110,7 +1110,7 @@ describe('WebsocketController', function () { return (this.update = { doc_id: (this.doc_id = 'doc-id-123'), row: (this.row = 42), - column: (this.column = 37) + column: (this.column = 37), }) }) @@ -1121,7 +1121,7 @@ describe('WebsocketController', function () { first_name: (this.first_name = 'Douglas'), last_name: (this.last_name = 'Adams'), email: (this.email = 'joe@example.com'), - user_id: (this.user_id = 'user-id-123') + user_id: (this.user_id = 'user-id-123'), } this.populatedCursorData = { @@ -1131,7 +1131,7 @@ describe('WebsocketController', function () { row: this.row, column: this.column, email: this.email, - user_id: this.user_id + user_id: this.user_id, } this.WebsocketController.updateClientPosition( this.client, @@ -1159,12 +1159,12 @@ describe('WebsocketController', function () { _id: this.user_id, email: this.email, first_name: this.first_name, - last_name: this.last_name + last_name: this.last_name, }, { row: this.row, column: this.column, - doc_id: this.doc_id + doc_id: this.doc_id, } ) .should.equal(true) @@ -1185,7 +1185,7 @@ describe('WebsocketController', function () { first_name: (this.first_name = 'Douglas'), last_name: undefined, email: (this.email = 'joe@example.com'), - user_id: (this.user_id = 'user-id-123') + user_id: (this.user_id = 'user-id-123'), } this.populatedCursorData = { @@ -1195,7 +1195,7 @@ describe('WebsocketController', function () { row: this.row, column: this.column, email: this.email, - user_id: this.user_id + user_id: this.user_id, } this.WebsocketController.updateClientPosition( this.client, @@ -1223,12 +1223,12 @@ describe('WebsocketController', function () { _id: this.user_id, email: this.email, first_name: this.first_name, - last_name: undefined + last_name: undefined, }, { row: this.row, column: this.column, - doc_id: this.doc_id + doc_id: this.doc_id, } ) .should.equal(true) @@ -1249,7 +1249,7 @@ describe('WebsocketController', function () { first_name: undefined, last_name: (this.last_name = 'Adams'), email: (this.email = 'joe@example.com'), - user_id: (this.user_id = 'user-id-123') + user_id: (this.user_id = 'user-id-123'), } this.populatedCursorData = { @@ -1259,7 +1259,7 @@ describe('WebsocketController', function () { row: this.row, column: this.column, email: this.email, - user_id: this.user_id + user_id: this.user_id, } this.WebsocketController.updateClientPosition( this.client, @@ -1287,12 +1287,12 @@ describe('WebsocketController', function () { _id: this.user_id, email: this.email, first_name: undefined, - last_name: this.last_name + last_name: this.last_name, }, { row: this.row, column: this.column, - doc_id: this.doc_id + doc_id: this.doc_id, } ) .should.equal(true) @@ -1312,7 +1312,7 @@ describe('WebsocketController', function () { first_name: undefined, last_name: undefined, email: (this.email = 'joe@example.com'), - user_id: (this.user_id = 'user-id-123') + user_id: (this.user_id = 'user-id-123'), } return this.WebsocketController.updateClientPosition( this.client, @@ -1330,7 +1330,7 @@ describe('WebsocketController', function () { name: '', row: this.row, column: this.column, - email: this.email + email: this.email, }) .should.equal(true) }) @@ -1339,7 +1339,7 @@ describe('WebsocketController', function () { describe('with an anonymous user', function () { beforeEach(function (done) { this.client.ol_context = { - project_id: this.project_id + project_id: this.project_id, } return this.WebsocketController.updateClientPosition( this.client, @@ -1355,7 +1355,7 @@ describe('WebsocketController', function () { id: this.client.publicId, name: '', row: this.row, - column: this.column + column: this.column, }) .should.equal(true) }) @@ -1369,7 +1369,8 @@ describe('WebsocketController', function () { return describe('when the client has disconnected', function () { beforeEach(function (done) { this.client.disconnected = true - this.AuthorizationManager.assertClientCanViewProjectAndDoc = sinon.stub() + this.AuthorizationManager.assertClientCanViewProjectAndDoc = + sinon.stub() return this.WebsocketController.updateClientPosition( this.client, this.update, @@ -1526,9 +1527,9 @@ describe('WebsocketController', function () { user_id: this.user_id, project_id: this.project_id, doc_id: this.doc_id, - updateSize: 7372835 + updateSize: 7372835, }, - 'update is too large' + 'update is too large', ]) }) @@ -1565,7 +1566,7 @@ describe('WebsocketController', function () { return it('should increment the editor.doc-update.disconnected metric with a status', function () { return expect( this.metrics.inc.calledWith('editor.doc-update.disconnected', 1, { - status: 'at-otUpdateError' + status: 'at-otUpdateError', }) ).to.equal(true) }) @@ -1578,12 +1579,13 @@ describe('WebsocketController', function () { this.edit_update = { op: [ { i: 'foo', p: 42 }, - { c: 'bar', p: 132 } - ] + { c: 'bar', p: 132 }, + ], } // comments may still be in an edit op this.comment_update = { op: [{ c: 'bar', p: 132 }] } this.AuthorizationManager.assertClientCanEditProjectAndDoc = sinon.stub() - return (this.AuthorizationManager.assertClientCanViewProjectAndDoc = sinon.stub()) + return (this.AuthorizationManager.assertClientCanViewProjectAndDoc = + sinon.stub()) }) describe('with a read-write client', function () { @@ -1593,7 +1595,7 @@ describe('WebsocketController', function () { this.client, this.doc_id, this.edit_update, - (error) => { + error => { expect(error).to.be.null return done() } @@ -1611,7 +1613,7 @@ describe('WebsocketController', function () { this.client, this.doc_id, this.edit_update, - (error) => { + error => { expect(error.message).to.equal('not authorized') return done() } @@ -1629,7 +1631,7 @@ describe('WebsocketController', function () { this.client, this.doc_id, this.comment_update, - (error) => { + error => { expect(error).to.be.null return done() } @@ -1649,7 +1651,7 @@ describe('WebsocketController', function () { this.client, this.doc_id, this.comment_update, - (error) => { + error => { expect(error.message).to.equal('not authorized') return done() } diff --git a/services/real-time/test/unit/js/WebsocketLoadBalancerTests.js b/services/real-time/test/unit/js/WebsocketLoadBalancerTests.js index 9e61501305..ec341a43bd 100644 --- a/services/real-time/test/unit/js/WebsocketLoadBalancerTests.js +++ b/services/real-time/test/unit/js/WebsocketLoadBalancerTests.js @@ -23,29 +23,29 @@ describe('WebsocketLoadBalancer', function () { this.WebsocketLoadBalancer = SandboxedModule.require(modulePath, { requires: { './RedisClientManager': { - createClientList: () => [] + createClientList: () => [], }, './SafeJsonParse': (this.SafeJsonParse = { - parse: (data, cb) => cb(null, JSON.parse(data)) + parse: (data, cb) => cb(null, JSON.parse(data)), }), './EventLogger': { checkEventOrder: sinon.stub() }, './HealthCheckManager': { check: sinon.stub() }, './RoomManager': (this.RoomManager = { - eventSource: sinon.stub().returns(this.RoomEvents) + eventSource: sinon.stub().returns(this.RoomEvents), }), './ChannelManager': (this.ChannelManager = { publish: sinon.stub() }), './ConnectedUsersManager': (this.ConnectedUsersManager = { - refreshClient: sinon.stub() - }) - } + refreshClient: sinon.stub(), + }), + }, }) this.io = {} this.WebsocketLoadBalancer.rclientPubList = [{ publish: sinon.stub() }] this.WebsocketLoadBalancer.rclientSubList = [ { subscribe: sinon.stub(), - on: sinon.stub() - } + on: sinon.stub(), + }, ] this.room_id = 'room-id' @@ -71,7 +71,7 @@ describe('WebsocketLoadBalancer', function () { JSON.stringify({ room_id: this.room_id, message: this.message, - payload: this.payload + payload: this.payload, }) ) .should.equal(true) @@ -139,24 +139,24 @@ describe('WebsocketLoadBalancer', function () { { id: 'client-id-1', emit: (this.emit1 = sinon.stub()), - ol_context: {} + ol_context: {}, }, { id: 'client-id-2', emit: (this.emit2 = sinon.stub()), - ol_context: {} + ol_context: {}, }, { id: 'client-id-1', emit: (this.emit3 = sinon.stub()), - ol_context: {} - } // duplicate client - ]) + ol_context: {}, + }, // duplicate client + ]), } const data = JSON.stringify({ room_id: this.room_id, message: this.message, - payload: this.payload + payload: this.payload, }) return this.WebsocketLoadBalancer._processEditorEvent( this.io, @@ -184,29 +184,29 @@ describe('WebsocketLoadBalancer', function () { { id: 'client-id-1', emit: (this.emit1 = sinon.stub()), - ol_context: {} + ol_context: {}, }, { id: 'client-id-2', emit: (this.emit2 = sinon.stub()), - ol_context: {} + ol_context: {}, }, { id: 'client-id-1', emit: (this.emit3 = sinon.stub()), - ol_context: {} + ol_context: {}, }, // duplicate client { id: 'client-id-4', emit: (this.emit4 = sinon.stub()), - ol_context: { is_restricted_user: true } - } - ]) + ol_context: { is_restricted_user: true }, + }, + ]), } const data = JSON.stringify({ room_id: this.room_id, message: this.message, - payload: this.payload + payload: this.payload, }) return this.WebsocketLoadBalancer._processEditorEvent( this.io, @@ -235,29 +235,29 @@ describe('WebsocketLoadBalancer', function () { { id: 'client-id-1', emit: (this.emit1 = sinon.stub()), - ol_context: {} + ol_context: {}, }, { id: 'client-id-2', emit: (this.emit2 = sinon.stub()), - ol_context: {} + ol_context: {}, }, { id: 'client-id-1', emit: (this.emit3 = sinon.stub()), - ol_context: {} + ol_context: {}, }, // duplicate client { id: 'client-id-4', emit: (this.emit4 = sinon.stub()), - ol_context: { is_restricted_user: true } - } - ]) + ol_context: { is_restricted_user: true }, + }, + ]), } const data = JSON.stringify({ room_id: this.room_id, message: (this.restrictedMessage = 'new-comment'), - payload: this.payload + payload: this.payload, }) return this.WebsocketLoadBalancer._processEditorEvent( this.io, @@ -285,7 +285,7 @@ describe('WebsocketLoadBalancer', function () { const data = JSON.stringify({ room_id: 'all', message: this.message, - payload: this.payload + payload: this.payload, }) return this.WebsocketLoadBalancer._processEditorEvent( this.io, From 9e4624315bd1042ee6976e35e285eb010cf06f29 Mon Sep 17 00:00:00 2001 From: Jakob Ackermann Date: Tue, 13 Jul 2021 12:21:10 +0100 Subject: [PATCH 488/491] [misc] temporary override a few new/changed eslint rules --- services/real-time/.eslintrc | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) diff --git a/services/real-time/.eslintrc b/services/real-time/.eslintrc index 1c14f50efe..a97661b15f 100644 --- a/services/real-time/.eslintrc +++ b/services/real-time/.eslintrc @@ -5,7 +5,7 @@ "extends": [ "eslint:recommended", "standard", - "prettier", + "prettier" ], "parserOptions": { "ecmaVersion": 2018 @@ -20,6 +20,19 @@ "mocha": true }, "rules": { + // TODO(das7pad): remove overrides after fixing all the violations manually (https://github.com/overleaf/issues/issues/3882#issuecomment-878999671) + // START of temporary overrides + "array-callback-return": "off", + "no-dupe-else-if": "off", + "no-var": "off", + "no-empty": "off", + "node/handle-callback-err": "off", + "no-loss-of-precision": "off", + "node/no-callback-literal": "off", + "node/no-path-concat": "off", + "prefer-regex-literals": "off", + // END of temporary overrides + // Swap the no-unused-expressions rule with a more chai-friendly one "no-unused-expressions": 0, "chai-friendly/no-unused-expressions": "error", From 89928dd19db5323dc3f7590fd70ce144c28ee475 Mon Sep 17 00:00:00 2001 From: Jakob Ackermann Date: Tue, 13 Jul 2021 12:26:33 +0100 Subject: [PATCH 489/491] [misc] upgrade node version to latest v12 LTS version 12.22.3 --- services/real-time/.nvmrc | 2 +- services/real-time/Dockerfile | 2 +- services/real-time/buildscript.txt | 2 +- services/real-time/docker-compose.yml | 4 ++-- 4 files changed, 5 insertions(+), 5 deletions(-) diff --git a/services/real-time/.nvmrc b/services/real-time/.nvmrc index e68b860383..5a80a7e912 100644 --- a/services/real-time/.nvmrc +++ b/services/real-time/.nvmrc @@ -1 +1 @@ -12.21.0 +12.22.3 diff --git a/services/real-time/Dockerfile b/services/real-time/Dockerfile index 4f417a2a4b..6b286376dc 100644 --- a/services/real-time/Dockerfile +++ b/services/real-time/Dockerfile @@ -2,7 +2,7 @@ # Instead run bin/update_build_scripts from # https://github.com/sharelatex/sharelatex-dev-environment -FROM node:12.21.0 as base +FROM node:12.22.3 as base WORKDIR /app diff --git a/services/real-time/buildscript.txt b/services/real-time/buildscript.txt index e536d601f3..fcf340d99c 100644 --- a/services/real-time/buildscript.txt +++ b/services/real-time/buildscript.txt @@ -3,6 +3,6 @@ real-time --docker-repos=gcr.io/overleaf-ops --env-add= --env-pass-through= ---node-version=12.21.0 +--node-version=12.22.3 --public-repo=True --script-version=3.11.0 diff --git a/services/real-time/docker-compose.yml b/services/real-time/docker-compose.yml index ad04d1ab54..59ad1c5efe 100644 --- a/services/real-time/docker-compose.yml +++ b/services/real-time/docker-compose.yml @@ -6,7 +6,7 @@ version: "2.3" services: test_unit: - image: node:12.21.0 + image: node:12.22.3 volumes: - .:/app working_dir: /app @@ -18,7 +18,7 @@ services: user: node test_acceptance: - image: node:12.21.0 + image: node:12.22.3 volumes: - .:/app working_dir: /app From bd5e8b8f7160f66594f89becad1e71099dd8f2d3 Mon Sep 17 00:00:00 2001 From: Jakob Ackermann Date: Tue, 13 Jul 2021 12:34:56 +0100 Subject: [PATCH 490/491] [misc] fix chai assertions .equal.true -> .equals(true) --- .../real-time/test/unit/js/EventLoggerTests.js | 17 +++++++++-------- 1 file changed, 9 insertions(+), 8 deletions(-) diff --git a/services/real-time/test/unit/js/EventLoggerTests.js b/services/real-time/test/unit/js/EventLoggerTests.js index f647741fcf..fd759c8b42 100644 --- a/services/real-time/test/unit/js/EventLoggerTests.js +++ b/services/real-time/test/unit/js/EventLoggerTests.js @@ -54,8 +54,9 @@ describe('EventLogger', function () { }) return it('should increment the valid event metric', function () { - return this.metrics.inc.calledWith(`event.${this.channel}.valid`, 1) - .should.equal.true + return this.metrics.inc + .calledWith(`event.${this.channel}.valid`, 1) + .should.equals(true) }) }) @@ -78,8 +79,9 @@ describe('EventLogger', function () { }) return it('should increment the duplicate event metric', function () { - return this.metrics.inc.calledWith(`event.${this.channel}.duplicate`, 1) - .should.equal.true + return this.metrics.inc + .calledWith(`event.${this.channel}.duplicate`, 1) + .should.equals(true) }) }) @@ -107,10 +109,9 @@ describe('EventLogger', function () { }) return it('should increment the out-of-order event metric', function () { - return this.metrics.inc.calledWith( - `event.${this.channel}.out-of-order`, - 1 - ).should.equal.true + return this.metrics.inc + .calledWith(`event.${this.channel}.out-of-order`, 1) + .should.equals(true) }) }) From b8fcb265b2f2f5dbd6f915ff510635a7b6c531ff Mon Sep 17 00:00:00 2001 From: Jakob Ackermann Date: Tue, 13 Jul 2021 12:40:46 +0100 Subject: [PATCH 491/491] [misc] EventLogger: drop explicit metrics.inc amount The module has a hard-coded increment of 1, which is the only value the prometheus backend supports. --- services/real-time/app/js/EventLogger.js | 2 +- services/real-time/test/unit/js/EventLoggerTests.js | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/services/real-time/app/js/EventLogger.js b/services/real-time/app/js/EventLogger.js index 21c3460865..91f63e143c 100644 --- a/services/real-time/app/js/EventLogger.js +++ b/services/real-time/app/js/EventLogger.js @@ -43,7 +43,7 @@ module.exports = EventLogger = { // store the last count in a hash for each host const previous = EventLogger._storeEventCount(key, count) if (!previous || count === previous + 1) { - metrics.inc(`event.${channel}.valid`, 0.001) // downsample high rate docupdater events + metrics.inc(`event.${channel}.valid`) return // order is ok } if (count === previous) { diff --git a/services/real-time/test/unit/js/EventLoggerTests.js b/services/real-time/test/unit/js/EventLoggerTests.js index fd759c8b42..037f2e214a 100644 --- a/services/real-time/test/unit/js/EventLoggerTests.js +++ b/services/real-time/test/unit/js/EventLoggerTests.js @@ -55,7 +55,7 @@ describe('EventLogger', function () { return it('should increment the valid event metric', function () { return this.metrics.inc - .calledWith(`event.${this.channel}.valid`, 1) + .calledWith(`event.${this.channel}.valid`) .should.equals(true) }) }) @@ -80,7 +80,7 @@ describe('EventLogger', function () { return it('should increment the duplicate event metric', function () { return this.metrics.inc - .calledWith(`event.${this.channel}.duplicate`, 1) + .calledWith(`event.${this.channel}.duplicate`) .should.equals(true) }) }) @@ -110,7 +110,7 @@ describe('EventLogger', function () { return it('should increment the out-of-order event metric', function () { return this.metrics.inc - .calledWith(`event.${this.channel}.out-of-order`, 1) + .calledWith(`event.${this.channel}.out-of-order`) .should.equals(true) }) })