diff --git a/services/spelling/.dockerignore b/services/spelling/.dockerignore index 386f26df30..ba1c3442de 100644 --- a/services/spelling/.dockerignore +++ b/services/spelling/.dockerignore @@ -5,5 +5,3 @@ gitrev .npm .nvmrc nodemon.json -app.js -**/js/* diff --git a/services/spelling/.eslintrc b/services/spelling/.eslintrc new file mode 100644 index 0000000000..dd8590b563 --- /dev/null +++ b/services/spelling/.eslintrc @@ -0,0 +1,36 @@ +// this file was auto-generated, do not edit it directly. +// instead run bin/update_build_scripts from +// https://github.com/sharelatex/sharelatex-dev-environment +// Version: 1.1.21 +{ + "extends": [ + "standard", + "prettier", + "prettier/standard", + ], + "plugins": [ + "mocha", + "chai-expect", + "chai-friendly" + ], + "env": { + "node": true + }, + "rules": { + // Add some mocha specific rules + "mocha/handle-done-callback": "error", + "mocha/no-exclusive-tests": "error", + "mocha/no-global-tests": "error", + "mocha/no-identical-title": "error", + "mocha/no-nested-tests": "error", + "mocha/no-pending-tests": "error", + "mocha/no-skipped-tests": "error", + + // Add some chai specific rules + "chai-expect/missing-assertion": "error", + "chai-expect/terminating-properties": "error", + // Swap the no-unused-expressions rule with a more chai-friendly one + "no-unused-expressions": 0, + "chai-friendly/no-unused-expressions": "error" + } +} diff --git a/services/spelling/.gitignore b/services/spelling/.gitignore index ad44901650..7c663ba608 100644 --- a/services/spelling/.gitignore +++ b/services/spelling/.gitignore @@ -1,9 +1,5 @@ **.swp **.swo -app/js/* -app.js -test/UnitTests/js/* node_modules/* -test/unit/js/ cache/spell.cache **/*.map diff --git a/services/spelling/.prettierrc b/services/spelling/.prettierrc new file mode 100644 index 0000000000..db87853525 --- /dev/null +++ b/services/spelling/.prettierrc @@ -0,0 +1,8 @@ +# This file was auto-generated, do not edit it directly. +# Instead run bin/update_build_scripts from +# https://github.com/sharelatex/sharelatex-dev-environment +# Version: 1.1.21 +{ + "semi": false, + "singleQuote": true +} diff --git a/services/spelling/Dockerfile b/services/spelling/Dockerfile index cfb417387e..2510521e29 100644 --- a/services/spelling/Dockerfile +++ b/services/spelling/Dockerfile @@ -10,7 +10,6 @@ RUN npm install --quiet COPY . /app -RUN npm run compile:all FROM node:6.16.0 diff --git a/services/spelling/Jenkinsfile b/services/spelling/Jenkinsfile index 8091c12db8..8a7cc4fd0b 100644 --- a/services/spelling/Jenkinsfile +++ b/services/spelling/Jenkinsfile @@ -36,6 +36,14 @@ pipeline { } } + // should be enabled once Node version is updated to >=8 + // stage('Linting') { + // steps { + // sh 'DOCKER_COMPOSE_FLAGS="-f docker-compose.ci.yml" make format' + // sh 'DOCKER_COMPOSE_FLAGS="-f docker-compose.ci.yml" make lint' + // } + // } + stage('Unit Tests') { steps { sh 'DOCKER_COMPOSE_FLAGS="-f docker-compose.ci.yml" make test_unit' diff --git a/services/spelling/Makefile b/services/spelling/Makefile index c764bba400..c7039c6b63 100644 --- a/services/spelling/Makefile +++ b/services/spelling/Makefile @@ -16,12 +16,17 @@ DOCKER_COMPOSE := BUILD_NUMBER=$(BUILD_NUMBER) \ clean: docker rmi ci/$(PROJECT_NAME):$(BRANCH_NAME)-$(BUILD_NUMBER) docker rmi gcr.io/overleaf-ops/$(PROJECT_NAME):$(BRANCH_NAME)-$(BUILD_NUMBER) - rm -f app.js - rm -rf app/js - rm -rf test/unit/js - rm -rf test/acceptance/js -test: test_unit test_acceptance +format: + $(DOCKER_COMPOSE) run --rm test_unit npm run format + +format_fix: + $(DOCKER_COMPOSE) run --rm test_unit npm run format:fix + +lint: + $(DOCKER_COMPOSE) run --rm test_unit npm run lint + +test: format lint test_unit test_acceptance test_unit: @[ ! -d test/unit ] && echo "spelling has no unit tests" || $(DOCKER_COMPOSE) run --rm test_unit diff --git a/services/spelling/app.coffee b/services/spelling/app.coffee deleted file mode 100644 index be1a76c1ba..0000000000 --- a/services/spelling/app.coffee +++ /dev/null @@ -1,47 +0,0 @@ -metrics = require("metrics-sharelatex") -metrics.initialize("spelling") - - -Settings = require 'settings-sharelatex' -logger = require 'logger-sharelatex' -logger.initialize("spelling") -if Settings.sentry?.dsn? - logger.initializeErrorReporting(Settings.sentry.dsn) -metrics.memory.monitor(logger) - -SpellingAPIController = require './app/js/SpellingAPIController' -express = require('express') -Path = require("path") -server = express() -metrics.injectMetricsRoute(server) -bodyParser = require('body-parser') -HealthCheckController = require("./app/js/HealthCheckController") - - - -server.use bodyParser.json(limit: "2mb") -server.use metrics.http.monitor(logger) - -server.del "/user/:user_id", SpellingAPIController.deleteDic -server.get "/user/:user_id", SpellingAPIController.getDic -server.post "/user/:user_id/check", SpellingAPIController.check -server.post "/user/:user_id/learn", SpellingAPIController.learn -server.get "/status", (req, res)-> - res.send(status:'spelling api is up') - -server.get "/health_check", HealthCheckController.healthCheck - -profiler = require "v8-profiler" -server.get "/profile", (req, res) -> - time = parseInt(req.query.time || "1000") - profiler.startProfiling("test") - setTimeout () -> - profile = profiler.stopProfiling("test") - res.json(profile) - , time - -host = Settings.internal?.spelling?.host || "localhost" -port = Settings.internal?.spelling?.port || 3005 -server.listen port, host, (error) -> - throw error if error? - logger.info "spelling starting up, listening on #{host}:#{port}" diff --git a/services/spelling/app.js b/services/spelling/app.js new file mode 100644 index 0000000000..003754e572 --- /dev/null +++ b/services/spelling/app.js @@ -0,0 +1,68 @@ +/* + * decaffeinate suggestions: + * DS102: Remove unnecessary code created because of implicit returns + * DS103: Rewrite code to no longer use __guard__ + * DS207: Consider shorter variations of null checks + * Full docs: https://github.com/decaffeinate/decaffeinate/blob/master/docs/suggestions.md + */ +const metrics = require('metrics-sharelatex') +metrics.initialize('spelling') + +const Settings = require('settings-sharelatex') +const logger = require('logger-sharelatex') +logger.initialize('spelling') +if ((Settings.sentry != null ? Settings.sentry.dsn : undefined) != null) { + logger.initializeErrorReporting(Settings.sentry.dsn) +} +metrics.memory.monitor(logger) + +const SpellingAPIController = require('./app/js/SpellingAPIController') +const express = require('express') +const server = express() +metrics.injectMetricsRoute(server) +const bodyParser = require('body-parser') +const HealthCheckController = require('./app/js/HealthCheckController') + +server.use(bodyParser.json({ limit: '2mb' })) +server.use(metrics.http.monitor(logger)) + +server.del('/user/:user_id', SpellingAPIController.deleteDic) +server.get('/user/:user_id', SpellingAPIController.getDic) +server.post('/user/:user_id/check', SpellingAPIController.check) +server.post('/user/:user_id/learn', SpellingAPIController.learn) +server.get('/status', (req, res) => res.send({ status: 'spelling api is up' })) + +server.get('/health_check', HealthCheckController.healthCheck) + +const profiler = require('v8-profiler') +server.get('/profile', function(req, res) { + const time = parseInt(req.query.time || '1000') + profiler.startProfiling('test') + return setTimeout(function() { + const profile = profiler.stopProfiling('test') + return res.json(profile) + }, time) +}) + +const host = + __guard__( + Settings.internal != null ? Settings.internal.spelling : undefined, + x => x.host + ) || 'localhost' +const port = + __guard__( + Settings.internal != null ? Settings.internal.spelling : undefined, + x1 => x1.port + ) || 3005 +server.listen(port, host, function(error) { + if (error != null) { + throw error + } + return logger.info(`spelling starting up, listening on ${host}:${port}`) +}) + +function __guard__(value, transform) { + return typeof value !== 'undefined' && value !== null + ? transform(value) + : undefined +} diff --git a/services/spelling/app/coffee/ASpell.coffee b/services/spelling/app/coffee/ASpell.coffee deleted file mode 100644 index 8f70ab19f8..0000000000 --- a/services/spelling/app/coffee/ASpell.coffee +++ /dev/null @@ -1,119 +0,0 @@ -async = require "async" -ASpellWorkerPool = require "./ASpellWorkerPool" -LRU = require "lru-cache" -logger = require 'logger-sharelatex' -fs = require 'fs' -settings = require("settings-sharelatex") -Path = require("path") - -OneMinute = 60 * 1000 -opts = {max:10000, maxAge: OneMinute * 60 * 10} -cache = LRU(opts) - -cacheFsPath = Path.resolve(settings.cacheDir, "spell.cache") -cacheFsPathTmp = cacheFsPath + ".tmp" - -# load any existing cache -try - oldCache = fs.readFileSync cacheFsPath - cache.load JSON.parse(oldCache) -catch err - logger.log err:err, cacheFsPath:cacheFsPath, "could not load the cache file" - -# write the cache every 30 minutes -setInterval () -> - dump = JSON.stringify cache.dump() - fs.writeFile cacheFsPathTmp, dump, (err) -> - if err? - logger.log {err}, "error writing cache file" - fs.unlink cacheFsPathTmp - else - fs.rename cacheFsPathTmp, cacheFsPath - logger.log {len: dump.length, cacheFsPath:cacheFsPath}, "wrote cache file" -, 30 * OneMinute - -class ASpellRunner - checkWords: (language, words, callback = (error, result) ->) -> - @runAspellOnWords language, words, (error, output) => - return callback(error) if error? - #output = @removeAspellHeader(output) - suggestions = @getSuggestions(language, output) - results = [] - hits = 0 - addToCache = {} - for word, i in words - key = language + ':' + word - cached = cache.get(key) - if cached? - hits++ - if cached == true - # valid word, no need to do anything - continue - else - results.push index: i, suggestions: cached - else - if suggestions[key]? - addToCache[key] = suggestions[key] - results.push index: i, suggestions: suggestions[key] - else - # a valid word, but uncached - addToCache[key] = true - - # update the cache after processing all words, to avoid cache - # changing while we use it - for k, v of addToCache - cache.set(k, v) - - logger.info hits: hits, total: words.length, hitrate: (hits/words.length).toFixed(2), "cache hit rate" - callback null, results - - getSuggestions: (language, output) -> - lines = output.split("\n") - suggestions = {} - for line in lines - if line[0] == "&" # Suggestions found - parts = line.split(" ") - if parts.length > 1 - word = parts[1] - suggestionsString = line.slice(line.indexOf(":") + 2) - suggestions[language + ":" + word] = suggestionsString.split(", ") - else if line[0] == "#" # No suggestions - parts = line.split(" ") - if parts.length > 1 - word = parts[1] - suggestions[language + ":" + word] = [] - return suggestions - - #removeAspellHeader: (output) -> output.slice(1) - - runAspellOnWords: (language, words, callback = (error, output) ->) -> - # send words to aspell, get back string output for those words - # find a free pipe for the language (or start one) - # send the words down the pipe - # send an END marker that will generate a "*" line in the output - # when the output pipe receives the "*" return the data sofar and reset the pipe to be available - # - # @open(language) - # @captureOutput(callback) - # @setTerseMode() - # start = new Date() - - newWord = {} - for word in words - newWord[word] = true if !newWord[word] && !cache.has(language + ':' + word) - words = Object.keys(newWord) - - if words.length - WorkerPool.check(language, words, ASpell.ASPELL_TIMEOUT, callback) - else - callback null, "" - -module.exports = ASpell = - # The description of how to call aspell from another program can be found here: - # http://aspell.net/man-html/Through-A-Pipe.html - checkWords: (language, words, callback = (error, result) ->) -> - runner = new ASpellRunner() - runner.checkWords language, words, callback - ASPELL_TIMEOUT : 4000 - -WorkerPool = new ASpellWorkerPool() diff --git a/services/spelling/app/coffee/ASpellWorker.coffee b/services/spelling/app/coffee/ASpellWorker.coffee deleted file mode 100644 index 2c919b721c..0000000000 --- a/services/spelling/app/coffee/ASpellWorker.coffee +++ /dev/null @@ -1,117 +0,0 @@ -child_process = require("child_process") -logger = require 'logger-sharelatex' -metrics = require('metrics-sharelatex') -_ = require "underscore" - -BATCH_SIZE = 100 - -class ASpellWorker - constructor: (language) -> - @language = language - @count = 0 - @pipe = child_process.spawn("aspell", ["pipe", "-t", "--encoding=utf-8", "-d", language]) - logger.info process: @pipe.pid, lang: @language, "starting new aspell worker" - metrics.inc "aspellWorker", 1, {status: "start", method: @language} - @pipe.on 'exit', () => - @state = 'killed' - logger.info process: @pipe.pid, lang: @language, "aspell worker has exited" - metrics.inc "aspellWorker" , 1, {status: "exit", method: @language} - @pipe.on 'close', () => - @state = 'closed' unless @state == 'killed' - if @callback? - logger.err process: @pipe.pid, lang: @language, "aspell worker closed output streams with uncalled callback" - @callback new Error("aspell worker closed output streams with uncalled callback"), [] - @callback = null - @pipe.on 'error', (err) => - @state = 'error' unless @state == 'killed' - logger.log process: @pipe.pid, error: err, stdout: output.slice(-1024), stderr: error.slice(-1024), lang: @language, "aspell worker error" - if @callback? - @callback err, [] - @callback = null - @pipe.stdin.on 'error', (err) => - @state = 'error' unless @state == 'killed' - logger.info process: @pipe.pid, error: err, stdout: output.slice(-1024), stderr: error.slice(-1024), lang: @language, "aspell worker error on stdin" - if @callback? - @callback err, [] - @callback = null - - output = "" - endMarker = new RegExp("^[a-z][a-z]", "m") - @pipe.stdout.on "data", (chunk) => - output = output + chunk - # We receive the language code from Aspell as the end of data marker - if chunk.toString().match(endMarker) - if @callback? - @callback(null, output.slice()) - @callback = null # only allow one callback in use - else - logger.err process: @pipe.pid, lang: @language, "end of data marker received when callback already used" - @state = 'ready' - output = "" - error = "" - - error = "" - @pipe.stderr.on "data", (chunk) => - error = error + chunk - - @pipe.stdout.on "end", () => - # process has ended - @state = "end" - - isReady: () -> - return @state == 'ready' - - check: (words, callback) -> - # we will now send data to aspell, and be ready again when we - # receive the end of data marker - @state = 'busy' - if @callback? # only allow one callback in use - logger.err process: @pipe.pid, lang: @language, "CALLBACK ALREADY IN USE" - return @callback new Error("Aspell callback already in use - SHOULD NOT HAPPEN") - @callback = _.once callback # extra defence against double callback - @setTerseMode() - @write(words) - @flush() - - write: (words) -> - i = 0 - while i < words.length - # batch up the words to check for efficiency - batch = words.slice(i, i + BATCH_SIZE) - @sendWords batch - i += BATCH_SIZE - - flush: () -> - # get aspell to send an end of data marker "*" when ready - #@sendCommand("%") # take the aspell pipe out of terse mode so we can look for a '*' - #@sendCommand("^ENDOFSTREAMMARKER") # send our marker which will generate a '*' - #@sendCommand("!") # go back into terse mode - @sendCommand("$$l") - - shutdown: (reason) -> - logger.info process: @pipe.pid, reason: reason, 'shutting down' - @state = "closing" - @pipe.stdin.end() - - kill: (reason) -> - logger.info process: @pipe.pid, reason: reason, 'killing' - return if @state == 'killed' - @pipe.kill('SIGKILL') - - setTerseMode: () -> - @sendCommand("!") - - sendWord: (word) -> - @sendCommand("^" + word) - - sendWords: (words) -> - # Aspell accepts multiple words to check on the same line - # ^word1 word2 word3 ... - # See aspell.info, writing programs to use Aspell Through A Pipe - @sendCommand("^" + words.join(" ")) - @count++ - - sendCommand: (command) -> - @pipe.stdin.write(command + "\n") - -module.exports = ASpellWorker diff --git a/services/spelling/app/coffee/ASpellWorkerPool.coffee b/services/spelling/app/coffee/ASpellWorkerPool.coffee deleted file mode 100644 index b8e7be4ac4..0000000000 --- a/services/spelling/app/coffee/ASpellWorkerPool.coffee +++ /dev/null @@ -1,76 +0,0 @@ -ASpellWorker = require "./ASpellWorker" -_ = require "underscore" -logger = require 'logger-sharelatex' -metrics = require('metrics-sharelatex') - -class ASpellWorkerPool - MAX_REQUESTS: 100*1024 - MAX_WORKERS: 32 - MAX_IDLE_TIME: 1000 - MAX_REQUEST_TIME: 60*1000 - - constructor: (@options) -> - @PROCESS_POOL = [] - - create: (language) -> - if @PROCESS_POOL.length >= @MAX_WORKERS - logger.log maxworkers: @MAX_WORKERS, "maximum number of workers already running" - return null - worker = new ASpellWorker(language, @options) - worker.pipe.on 'exit', () => - if worker.killTimer? - clearTimeout worker.killTimer - worker.killTimer = null - if worker.idleTimer? - clearTimeout worker.idleTimer - worker.idleTimer = null - logger.info process: worker.pipe.pid, lang: language, "removing aspell worker from pool" - @cleanup() - @PROCESS_POOL.push(worker) - metrics.gauge 'aspellWorkerPool-size', @PROCESS_POOL.length - return worker - - cleanup: () -> - active = @PROCESS_POOL.filter (worker) -> - worker.state != 'killed' - @PROCESS_POOL = active - metrics.gauge 'aspellWorkerPool-size', @PROCESS_POOL.length - - check: (language, words, timeout, callback) -> - # look for an existing process in the pool - availableWorker = _.find @PROCESS_POOL, (cached) -> - cached.language == language && cached.isReady() - if not availableWorker? - worker = @create(language) - else - worker = availableWorker - - if not worker? - # return error if too many workers - callback(new Error("no worker available")) - return - - if worker.idleTimer? - clearTimeout worker.idleTimer - worker.idleTimer = null - - worker.killTimer = setTimeout () -> - worker.kill("spell check timed out") - , timeout || @MAX_REQUEST_TIME - - worker.check words, (err, output) => - if worker.killTimer? - clearTimeout worker.killTimer - worker.killTimer = null - callback(err, output) - return if err? # process has shut down - if worker.count > @MAX_REQUESTS - worker.shutdown("reached limit of " + @MAX_REQUESTS + " requests") - else - # queue a shutdown if worker is idle - worker.idleTimer = setTimeout () -> - worker.shutdown("idle worker") - worker.idleTimer = null - , @MAX_IDLE_TIME - -module.exports = ASpellWorkerPool diff --git a/services/spelling/app/coffee/DB.coffee b/services/spelling/app/coffee/DB.coffee deleted file mode 100644 index be661fd2ce..0000000000 --- a/services/spelling/app/coffee/DB.coffee +++ /dev/null @@ -1,4 +0,0 @@ -MongoJS = require "mongojs" -Settings = require "settings-sharelatex" -module.exports = MongoJS(Settings.mongo.url, ["spellingPreferences"]) - diff --git a/services/spelling/app/coffee/HealthCheckController.coffee b/services/spelling/app/coffee/HealthCheckController.coffee deleted file mode 100644 index 504529bd93..0000000000 --- a/services/spelling/app/coffee/HealthCheckController.coffee +++ /dev/null @@ -1,23 +0,0 @@ -request = require("request") -logger = require 'logger-sharelatex' -settings = require 'settings-sharelatex' - -module.exports = - - healthCheck: (req, res)-> - opts = - url: "http://localhost:3005/user/#{settings.healthCheckUserId}/check" - json: - words:["helllo"] - language: "en" - timeout: 1000 * 20 - request.post opts, (err, response, body)-> - if err? - return res.sendStatus 500 - numberOfSuggestions = body?.misspellings?[0]?.suggestions?.length - if numberOfSuggestions > 10 - logger.log "health check passed" - res.sendStatus 200 - else - logger.err body:body, numberOfSuggestions:numberOfSuggestions, "health check failed" - res.sendStatus 500 diff --git a/services/spelling/app/coffee/LearnedWordsManager.coffee b/services/spelling/app/coffee/LearnedWordsManager.coffee deleted file mode 100644 index b0e0b38cc9..0000000000 --- a/services/spelling/app/coffee/LearnedWordsManager.coffee +++ /dev/null @@ -1,40 +0,0 @@ -db = require("./DB") -mongoCache = require("./MongoCache") -logger = require 'logger-sharelatex' -metrics = require('metrics-sharelatex') - -module.exports = LearnedWordsManager = - learnWord: (user_token, word, callback = (error)->) -> - mongoCache.del(user_token) - db.spellingPreferences.update { - token: user_token - }, { - $push: learnedWords: word - }, { - upsert: true - }, callback - - getLearnedWords: (user_token, callback = (error, words)->) -> - mongoCachedWords = mongoCache.get(user_token) - if mongoCachedWords? - metrics.inc "mongoCache", 0.1, {status: "hit"} - return callback(null, mongoCachedWords) - - metrics.inc "mongoCache", 0.1, {status: "miss"} - logger.info user_token:user_token, "mongoCache miss" - - db.spellingPreferences.findOne token: user_token, (error, preferences) -> - return callback error if error? - words = preferences?.learnedWords || [] - mongoCache.set(user_token, words) - callback null, words - - deleteUsersLearnedWords: (user_token, callback =(error)->)-> - db.spellingPreferences.remove token: user_token, callback - - -[ - 'learnWord', - 'getLearnedWords' -].map (method) -> - metrics.timeAsyncMethod(LearnedWordsManager, method, 'mongo.LearnedWordsManager', logger) diff --git a/services/spelling/app/coffee/MongoCache.coffee b/services/spelling/app/coffee/MongoCache.coffee deleted file mode 100644 index 8138c92696..0000000000 --- a/services/spelling/app/coffee/MongoCache.coffee +++ /dev/null @@ -1,8 +0,0 @@ -LRU = require("lru-cache") -cacheOpts = - max: 15000 - maxAge: 1000 * 60 * 60 * 10 - -cache = LRU(cacheOpts) - -module.exports = cache \ No newline at end of file diff --git a/services/spelling/app/coffee/SpellingAPIController.coffee b/services/spelling/app/coffee/SpellingAPIController.coffee deleted file mode 100644 index 1d2679d768..0000000000 --- a/services/spelling/app/coffee/SpellingAPIController.coffee +++ /dev/null @@ -1,35 +0,0 @@ -SpellingAPIManager = require './SpellingAPIManager' -logger = require 'logger-sharelatex' -metrics = require('metrics-sharelatex') - -module.exports = SpellingAPIController = - check: (req, res, next) -> - metrics.inc "spelling-check", 0.1 - logger.info token: req?.params?.user_id, word_count: req?.body?.words?.length, "running check" - SpellingAPIManager.runRequest req.params.user_id, req.body, (error, result) -> - if error? - logger.err err:error, user_id:req?.params?.user_id, word_count: req?.body?.words?.length, "error processing spelling request" - return res.sendStatus(500) - res.send(result) - - learn: (req, res, next) -> - metrics.inc "spelling-learn", 0.1 - logger.info token: req?.params?.user_id, word: req?.body?.word, "learning word" - SpellingAPIManager.learnWord req.params.user_id, req.body, (error, result) -> - return next(error) if error? - res.sendStatus(200) - next() - - - deleteDic: (req, res, next)-> - logger.log token: req?.params?.user_id, word: req?.body?.word, "deleting user dictionary" - SpellingAPIManager.deleteDic req.params.user_id, (error) -> - return next(error) if error? - res.sendStatus(204) - - - getDic: (req, res, next)-> - logger.info token: req?.params?.user_id, "getting user dictionary" - SpellingAPIManager.getDic req.params.user_id, (error, words)-> - return next(error) if error? - res.send(words) \ No newline at end of file diff --git a/services/spelling/app/coffee/SpellingAPIManager.coffee b/services/spelling/app/coffee/SpellingAPIManager.coffee deleted file mode 100644 index 9eb5cfc4aa..0000000000 --- a/services/spelling/app/coffee/SpellingAPIManager.coffee +++ /dev/null @@ -1,54 +0,0 @@ -ASpell = require './ASpell' -LearnedWordsManager = require './LearnedWordsManager' -async = require 'async' - -module.exports = SpellingAPIManager = - - whitelist: [ - 'ShareLaTeX', - 'sharelatex', - 'LaTeX', - 'http', - 'https', - 'www' - ] - - runRequest: (token, request, callback = (error, result) ->) -> - if !request.words? - return callback(new Error("malformed JSON")) - - lang = request.language || "en" - - check = (words, callback) -> - ASpell.checkWords lang, words, (error, misspellings) -> - callback error, misspellings: misspellings - - wordsToCheck = request.words || [] - - if token? - LearnedWordsManager.getLearnedWords token, (error, learnedWords) -> - return callback(error) if error? - words = (wordsToCheck).slice(0,10000) - check words, (error, result) -> - return callback error if error? - result.misspellings = result.misspellings.filter (m) -> - word = words[m.index] - learnedWords.indexOf(word) == -1 and SpellingAPIManager.whitelist.indexOf(word) == -1 - callback error, result - else - check(wordsToCheck, callback) - - learnWord: (token, request, callback = (error) ->) -> - if !request.word? - return callback(new Error("malformed JSON")) - if !token? - return callback(new Error("no token provided")) - - LearnedWordsManager.learnWord token, request.word, callback - - deleteDic: (token, callback)-> - LearnedWordsManager.deleteUsersLearnedWords token, callback - - getDic: (token, callback)-> - LearnedWordsManager.getLearnedWords token, callback - diff --git a/services/spelling/app/js/ASpell.js b/services/spelling/app/js/ASpell.js new file mode 100644 index 0000000000..b4421f2bd3 --- /dev/null +++ b/services/spelling/app/js/ASpell.js @@ -0,0 +1,175 @@ +// TODO: This file was created by bulk-decaffeinate. +// Sanity-check the conversion and remove this comment. +/* + * decaffeinate suggestions: + * DS101: Remove unnecessary use of Array.from + * DS102: Remove unnecessary code created because of implicit returns + * DS207: Consider shorter variations of null checks + * Full docs: https://github.com/decaffeinate/decaffeinate/blob/master/docs/suggestions.md + */ +let ASpell +const ASpellWorkerPool = require('./ASpellWorkerPool') +const LRU = require('lru-cache') +const logger = require('logger-sharelatex') +const fs = require('fs') +const settings = require('settings-sharelatex') +const Path = require('path') + +const OneMinute = 60 * 1000 +const opts = { max: 10000, maxAge: OneMinute * 60 * 10 } +const cache = LRU(opts) + +const cacheFsPath = Path.resolve(settings.cacheDir, 'spell.cache') +const cacheFsPathTmp = cacheFsPath + '.tmp' + +// load any existing cache +try { + const oldCache = fs.readFileSync(cacheFsPath) + cache.load(JSON.parse(oldCache)) +} catch (error) { + const err = error + logger.log({ err, cacheFsPath }, 'could not load the cache file') +} + +// write the cache every 30 minutes +setInterval(function() { + const dump = JSON.stringify(cache.dump()) + return fs.writeFile(cacheFsPathTmp, dump, function(err) { + if (err != null) { + logger.log({ err }, 'error writing cache file') + return fs.unlink(cacheFsPathTmp) + } else { + fs.rename(cacheFsPathTmp, cacheFsPath) + return logger.log({ len: dump.length, cacheFsPath }, 'wrote cache file') + } + }) +}, 30 * OneMinute) + +class ASpellRunner { + checkWords(language, words, callback) { + if (callback == null) { + callback = () => {} + } + return this.runAspellOnWords(language, words, (error, output) => { + if (error != null) { + return callback(error) + } + // output = @removeAspellHeader(output) + const suggestions = this.getSuggestions(language, output) + const results = [] + let hits = 0 + const addToCache = {} + for (let i = 0; i < words.length; i++) { + const word = words[i] + const key = language + ':' + word + const cached = cache.get(key) + if (cached != null) { + hits++ + if (cached === true) { + // valid word, no need to do anything + continue + } else { + results.push({ index: i, suggestions: cached }) + } + } else { + if (suggestions[key] != null) { + addToCache[key] = suggestions[key] + results.push({ index: i, suggestions: suggestions[key] }) + } else { + // a valid word, but uncached + addToCache[key] = true + } + } + } + + // update the cache after processing all words, to avoid cache + // changing while we use it + for (let k in addToCache) { + const v = addToCache[k] + cache.set(k, v) + } + + logger.info( + { + hits, + total: words.length, + hitrate: (hits / words.length).toFixed(2) + }, + 'cache hit rate' + ) + return callback(null, results) + }) + } + + getSuggestions(language, output) { + const lines = output.split('\n') + const suggestions = {} + for (let line of Array.from(lines)) { + var parts, word + if (line[0] === '&') { + // Suggestions found + parts = line.split(' ') + if (parts.length > 1) { + word = parts[1] + const suggestionsString = line.slice(line.indexOf(':') + 2) + suggestions[language + ':' + word] = suggestionsString.split(', ') + } + } else if (line[0] === '#') { + // No suggestions + parts = line.split(' ') + if (parts.length > 1) { + word = parts[1] + suggestions[language + ':' + word] = [] + } + } + } + return suggestions + } + + // removeAspellHeader: (output) -> output.slice(1) + + runAspellOnWords(language, words, callback) { + // send words to aspell, get back string output for those words + // find a free pipe for the language (or start one) + // send the words down the pipe + // send an END marker that will generate a "*" line in the output + // when the output pipe receives the "*" return the data sofar and reset the pipe to be available + // + // @open(language) + // @captureOutput(callback) + // @setTerseMode() + // start = new Date() + + if (callback == null) { + callback = () => {} + } + const newWord = {} + for (let word of Array.from(words)) { + if (!newWord[word] && !cache.has(language + ':' + word)) { + newWord[word] = true + } + } + words = Object.keys(newWord) + + if (words.length) { + return WorkerPool.check(language, words, ASpell.ASPELL_TIMEOUT, callback) + } else { + return callback(null, '') + } + } +} + +module.exports = ASpell = { + // The description of how to call aspell from another program can be found here: + // http://aspell.net/man-html/Through-A-Pipe.html + checkWords(language, words, callback) { + if (callback == null) { + callback = () => {} + } + const runner = new ASpellRunner() + return runner.checkWords(language, words, callback) + }, + ASPELL_TIMEOUT: 4000 +} + +var WorkerPool = new ASpellWorkerPool() diff --git a/services/spelling/app/js/ASpellWorker.js b/services/spelling/app/js/ASpellWorker.js new file mode 100644 index 0000000000..d691312ffe --- /dev/null +++ b/services/spelling/app/js/ASpellWorker.js @@ -0,0 +1,213 @@ +// TODO: This file was created by bulk-decaffeinate. +// Sanity-check the conversion and remove this comment. +/* + * decaffeinate suggestions: + * DS102: Remove unnecessary code created because of implicit returns + * DS205: Consider reworking code to avoid use of IIFEs + * DS207: Consider shorter variations of null checks + * Full docs: https://github.com/decaffeinate/decaffeinate/blob/master/docs/suggestions.md + */ +const childProcess = require('child_process') +const logger = require('logger-sharelatex') +const metrics = require('metrics-sharelatex') +const _ = require('underscore') + +const BATCH_SIZE = 100 + +class ASpellWorker { + constructor(language) { + this.language = language + this.count = 0 + this.pipe = childProcess.spawn('aspell', [ + 'pipe', + '-t', + '--encoding=utf-8', + '-d', + language + ]) + logger.info( + { process: this.pipe.pid, lang: this.language }, + 'starting new aspell worker' + ) + metrics.inc('aspellWorker', 1, { status: 'start', method: this.language }) + this.pipe.on('exit', () => { + this.state = 'killed' + logger.info( + { process: this.pipe.pid, lang: this.language }, + 'aspell worker has exited' + ) + return metrics.inc('aspellWorker', 1, { + status: 'exit', + method: this.language + }) + }) + this.pipe.on('close', () => { + if (this.state !== 'killed') { + this.state = 'closed' + } + if (this.callback != null) { + logger.err( + { process: this.pipe.pid, lang: this.language }, + 'aspell worker closed output streams with uncalled callback' + ) + this.callback( + new Error( + 'aspell worker closed output streams with uncalled callback' + ), + [] + ) + return (this.callback = null) + } + }) + this.pipe.on('error', err => { + if (this.state !== 'killed') { + this.state = 'error' + } + logger.log( + { + process: this.pipe.pid, + error: err, + stdout: output.slice(-1024), + stderr: error.slice(-1024), + lang: this.language + }, + 'aspell worker error' + ) + if (this.callback != null) { + this.callback(err, []) + return (this.callback = null) + } + }) + this.pipe.stdin.on('error', err => { + if (this.state !== 'killed') { + this.state = 'error' + } + logger.info( + { + process: this.pipe.pid, + error: err, + stdout: output.slice(-1024), + stderr: error.slice(-1024), + lang: this.language + }, + 'aspell worker error on stdin' + ) + if (this.callback != null) { + this.callback(err, []) + return (this.callback = null) + } + }) + + var output = '' + const endMarker = new RegExp('^[a-z][a-z]', 'm') + this.pipe.stdout.on('data', chunk => { + output = output + chunk + // We receive the language code from Aspell as the end of data marker + if (chunk.toString().match(endMarker)) { + if (this.callback != null) { + this.callback(null, output.slice()) + this.callback = null // only allow one callback in use + } else { + logger.err( + { process: this.pipe.pid, lang: this.language }, + 'end of data marker received when callback already used' + ) + } + this.state = 'ready' + output = '' + } + }) + + var error = '' + this.pipe.stderr.on('data', chunk => { + return (error = error + chunk) + }) + + this.pipe.stdout.on('end', () => { + // process has ended + return (this.state = 'end') + }) + } + + isReady() { + return this.state === 'ready' + } + + check(words, callback) { + // we will now send data to aspell, and be ready again when we + // receive the end of data marker + this.state = 'busy' + if (this.callback != null) { + // only allow one callback in use + logger.err( + { process: this.pipe.pid, lang: this.language }, + 'CALLBACK ALREADY IN USE' + ) + return this.callback( + new Error('Aspell callback already in use - SHOULD NOT HAPPEN') + ) + } + this.callback = _.once(callback) // extra defence against double callback + this.setTerseMode() + this.write(words) + return this.flush() + } + + write(words) { + let i = 0 + return (() => { + const result = [] + while (i < words.length) { + // batch up the words to check for efficiency + const batch = words.slice(i, i + BATCH_SIZE) + this.sendWords(batch) + result.push((i += BATCH_SIZE)) + } + return result + })() + } + + flush() { + // get aspell to send an end of data marker "*" when ready + // @sendCommand("%") # take the aspell pipe out of terse mode so we can look for a '*' + // @sendCommand("^ENDOFSTREAMMARKER") # send our marker which will generate a '*' + // @sendCommand("!") # go back into terse mode + return this.sendCommand('$$l') + } + + shutdown(reason) { + logger.info({ process: this.pipe.pid, reason }, 'shutting down') + this.state = 'closing' + return this.pipe.stdin.end() + } + + kill(reason) { + logger.info({ process: this.pipe.pid, reason }, 'killing') + if (this.state === 'killed') { + return + } + return this.pipe.kill('SIGKILL') + } + + setTerseMode() { + return this.sendCommand('!') + } + + sendWord(word) { + return this.sendCommand(`^${word}`) + } + + sendWords(words) { + // Aspell accepts multiple words to check on the same line + // ^word1 word2 word3 ... + // See aspell.info, writing programs to use Aspell Through A Pipe + this.sendCommand(`^${words.join(' ')}`) + return this.count++ + } + + sendCommand(command) { + return this.pipe.stdin.write(command + '\n') + } +} + +module.exports = ASpellWorker diff --git a/services/spelling/app/js/ASpellWorkerPool.js b/services/spelling/app/js/ASpellWorkerPool.js new file mode 100644 index 0000000000..a0daf8dee2 --- /dev/null +++ b/services/spelling/app/js/ASpellWorkerPool.js @@ -0,0 +1,115 @@ +// TODO: This file was created by bulk-decaffeinate. +// Sanity-check the conversion and remove this comment. +/* + * decaffeinate suggestions: + * DS102: Remove unnecessary code created because of implicit returns + * DS206: Consider reworking classes to avoid initClass + * DS207: Consider shorter variations of null checks + * Full docs: https://github.com/decaffeinate/decaffeinate/blob/master/docs/suggestions.md + */ +const ASpellWorker = require('./ASpellWorker') +const _ = require('underscore') +const logger = require('logger-sharelatex') +const metrics = require('metrics-sharelatex') + +class ASpellWorkerPool { + static initClass() { + this.prototype.MAX_REQUESTS = 100 * 1024 + this.prototype.MAX_WORKERS = 32 + this.prototype.MAX_IDLE_TIME = 1000 + this.prototype.MAX_REQUEST_TIME = 60 * 1000 + } + + constructor(options) { + this.options = options + this.PROCESS_POOL = [] + } + + create(language) { + if (this.PROCESS_POOL.length >= this.MAX_WORKERS) { + logger.log( + { maxworkers: this.MAX_WORKERS }, + 'maximum number of workers already running' + ) + return null + } + const worker = new ASpellWorker(language, this.options) + worker.pipe.on('exit', () => { + if (worker.killTimer != null) { + clearTimeout(worker.killTimer) + worker.killTimer = null + } + if (worker.idleTimer != null) { + clearTimeout(worker.idleTimer) + worker.idleTimer = null + } + logger.info( + { process: worker.pipe.pid, lang: language }, + 'removing aspell worker from pool' + ) + return this.cleanup() + }) + this.PROCESS_POOL.push(worker) + metrics.gauge('aspellWorkerPool-size', this.PROCESS_POOL.length) + return worker + } + + cleanup() { + const active = this.PROCESS_POOL.filter(worker => worker.state !== 'killed') + this.PROCESS_POOL = active + return metrics.gauge('aspellWorkerPool-size', this.PROCESS_POOL.length) + } + + check(language, words, timeout, callback) { + // look for an existing process in the pool + let worker + const availableWorker = _.find( + this.PROCESS_POOL, + cached => cached.language === language && cached.isReady() + ) + if (availableWorker == null) { + worker = this.create(language) + } else { + worker = availableWorker + } + + if (worker == null) { + // return error if too many workers + callback(new Error('no worker available')) + return + } + + if (worker.idleTimer != null) { + clearTimeout(worker.idleTimer) + worker.idleTimer = null + } + + worker.killTimer = setTimeout( + () => worker.kill('spell check timed out'), + timeout || this.MAX_REQUEST_TIME + ) + + return worker.check(words, (err, output) => { + if (worker.killTimer != null) { + clearTimeout(worker.killTimer) + worker.killTimer = null + } + callback(err, output) + if (err != null) { + return + } // process has shut down + if (worker.count > this.MAX_REQUESTS) { + return worker.shutdown(`reached limit of ${this.MAX_REQUESTS} requests`) + } else { + // queue a shutdown if worker is idle + return (worker.idleTimer = setTimeout(function() { + worker.shutdown('idle worker') + return (worker.idleTimer = null) + }, this.MAX_IDLE_TIME)) + } + }) + } +} +ASpellWorkerPool.initClass() + +module.exports = ASpellWorkerPool diff --git a/services/spelling/app/js/DB.js b/services/spelling/app/js/DB.js new file mode 100644 index 0000000000..416aa3b028 --- /dev/null +++ b/services/spelling/app/js/DB.js @@ -0,0 +1,5 @@ +// TODO: This file was created by bulk-decaffeinate. +// Sanity-check the conversion and remove this comment. +const MongoJS = require('mongojs') +const Settings = require('settings-sharelatex') +module.exports = MongoJS(Settings.mongo.url, ['spellingPreferences']) diff --git a/services/spelling/app/js/HealthCheckController.js b/services/spelling/app/js/HealthCheckController.js new file mode 100644 index 0000000000..5e32fa04e4 --- /dev/null +++ b/services/spelling/app/js/HealthCheckController.js @@ -0,0 +1,50 @@ +// TODO: This file was created by bulk-decaffeinate. +// Sanity-check the conversion and remove this comment. +/* + * decaffeinate suggestions: + * DS102: Remove unnecessary code created because of implicit returns + * DS103: Rewrite code to no longer use __guard__ + * DS207: Consider shorter variations of null checks + * Full docs: https://github.com/decaffeinate/decaffeinate/blob/master/docs/suggestions.md + */ +const request = require('request') +const logger = require('logger-sharelatex') +const settings = require('settings-sharelatex') + +module.exports = { + healthCheck(req, res) { + const opts = { + url: `http://localhost:3005/user/${settings.healthCheckUserId}/check`, + json: { + words: ['helllo'], + language: 'en' + }, + timeout: 1000 * 20 + } + return request.post(opts, function(err, response, body) { + if (err != null) { + return res.sendStatus(500) + } + const numberOfSuggestions = __guard__( + __guard__( + __guard__(body != null ? body.misspellings : undefined, x2 => x2[0]), + x1 => x1.suggestions + ), + x => x.length + ) + if (numberOfSuggestions > 10) { + logger.log('health check passed') + return res.sendStatus(200) + } else { + logger.err({ body, numberOfSuggestions }, 'health check failed') + return res.sendStatus(500) + } + }) + } +} + +function __guard__(value, transform) { + return typeof value !== 'undefined' && value !== null + ? transform(value) + : undefined +} diff --git a/services/spelling/app/js/LearnedWordsManager.js b/services/spelling/app/js/LearnedWordsManager.js new file mode 100644 index 0000000000..df7b614483 --- /dev/null +++ b/services/spelling/app/js/LearnedWordsManager.js @@ -0,0 +1,76 @@ +// TODO: This file was created by bulk-decaffeinate. +// Sanity-check the conversion and remove this comment. +/* + * decaffeinate suggestions: + * DS102: Remove unnecessary code created because of implicit returns + * DS207: Consider shorter variations of null checks + * Full docs: https://github.com/decaffeinate/decaffeinate/blob/master/docs/suggestions.md + */ +let LearnedWordsManager +const db = require('./DB') +const mongoCache = require('./MongoCache') +const logger = require('logger-sharelatex') +const metrics = require('metrics-sharelatex') + +module.exports = LearnedWordsManager = { + learnWord(userToken, word, callback) { + if (callback == null) { + callback = () => {} + } + mongoCache.del(userToken) + return db.spellingPreferences.update( + { + token: userToken + }, + { + $push: { learnedWords: word } + }, + { + upsert: true + }, + callback + ) + }, + + getLearnedWords(userToken, callback) { + if (callback == null) { + callback = () => {} + } + const mongoCachedWords = mongoCache.get(userToken) + if (mongoCachedWords != null) { + metrics.inc('mongoCache', 0.1, { status: 'hit' }) + return callback(null, mongoCachedWords) + } + + metrics.inc('mongoCache', 0.1, { status: 'miss' }) + logger.info({ userToken }, 'mongoCache miss') + + return db.spellingPreferences.findOne({ token: userToken }, function( + error, + preferences + ) { + if (error != null) { + return callback(error) + } + const words = + (preferences != null ? preferences.learnedWords : undefined) || [] + mongoCache.set(userToken, words) + return callback(null, words) + }) + }, + + deleteUsersLearnedWords(userToken, callback) { + if (callback == null) { + callback = () => {} + } + return db.spellingPreferences.remove({ token: userToken }, callback) + } +} +;['learnWord', 'getLearnedWords'].map(method => + metrics.timeAsyncMethod( + LearnedWordsManager, + method, + 'mongo.LearnedWordsManager', + logger + ) +) diff --git a/services/spelling/app/js/MongoCache.js b/services/spelling/app/js/MongoCache.js new file mode 100644 index 0000000000..7fedc77c0a --- /dev/null +++ b/services/spelling/app/js/MongoCache.js @@ -0,0 +1,11 @@ +// TODO: This file was created by bulk-decaffeinate. +// Sanity-check the conversion and remove this comment. +const LRU = require('lru-cache') +const cacheOpts = { + max: 15000, + maxAge: 1000 * 60 * 60 * 10 +} + +const cache = LRU(cacheOpts) + +module.exports = cache diff --git a/services/spelling/app/js/SpellingAPIController.js b/services/spelling/app/js/SpellingAPIController.js new file mode 100644 index 0000000000..5d357c3a77 --- /dev/null +++ b/services/spelling/app/js/SpellingAPIController.js @@ -0,0 +1,111 @@ +// TODO: This file was created by bulk-decaffeinate. +// Sanity-check the conversion and remove this comment. +/* + * decaffeinate suggestions: + * DS102: Remove unnecessary code created because of implicit returns + * DS103: Rewrite code to no longer use __guard__ + * DS207: Consider shorter variations of null checks + * Full docs: https://github.com/decaffeinate/decaffeinate/blob/master/docs/suggestions.md + */ +const SpellingAPIManager = require('./SpellingAPIManager') +const logger = require('logger-sharelatex') +const metrics = require('metrics-sharelatex') + +module.exports = { + check(req, res, next) { + metrics.inc('spelling-check', 0.1) + logger.info( + { + token: __guard__(req != null ? req.params : undefined, x => x.user_id), + word_count: __guard__( + __guard__(req != null ? req.body : undefined, x2 => x2.words), + x1 => x1.length + ) + }, + 'running check' + ) + return SpellingAPIManager.runRequest(req.params.user_id, req.body, function( + error, + result + ) { + if (error != null) { + logger.err( + { + err: error, + user_id: __guard__( + req != null ? req.params : undefined, + x3 => x3.user_id + ), + word_count: __guard__( + __guard__(req != null ? req.body : undefined, x5 => x5.words), + x4 => x4.length + ) + }, + 'error processing spelling request' + ) + return res.sendStatus(500) + } + return res.send(result) + }) + }, + + learn(req, res, next) { + metrics.inc('spelling-learn', 0.1) + logger.info( + { + token: __guard__(req != null ? req.params : undefined, x => x.user_id), + word: __guard__(req != null ? req.body : undefined, x1 => x1.word) + }, + 'learning word' + ) + return SpellingAPIManager.learnWord(req.params.user_id, req.body, function( + error, + result + ) { + if (error != null) { + return next(error) + } + res.sendStatus(200) + return next() + }) + }, + + deleteDic(req, res, next) { + logger.log( + { + token: __guard__(req != null ? req.params : undefined, x => x.user_id), + word: __guard__(req != null ? req.body : undefined, x1 => x1.word) + }, + 'deleting user dictionary' + ) + return SpellingAPIManager.deleteDic(req.params.user_id, function(error) { + if (error != null) { + return next(error) + } + return res.sendStatus(204) + }) + }, + + getDic(req, res, next) { + logger.info( + { + token: __guard__(req != null ? req.params : undefined, x => x.user_id) + }, + 'getting user dictionary' + ) + return SpellingAPIManager.getDic(req.params.user_id, function( + error, + words + ) { + if (error != null) { + return next(error) + } + return res.send(words) + }) + } +} +function __guard__(value, transform) { + return typeof value !== 'undefined' && value !== null + ? transform(value) + : undefined +} diff --git a/services/spelling/app/js/SpellingAPIManager.js b/services/spelling/app/js/SpellingAPIManager.js new file mode 100644 index 0000000000..ef127bec61 --- /dev/null +++ b/services/spelling/app/js/SpellingAPIManager.js @@ -0,0 +1,81 @@ +// TODO: This file was created by bulk-decaffeinate. +// Sanity-check the conversion and remove this comment. +/* + * decaffeinate suggestions: + * DS102: Remove unnecessary code created because of implicit returns + * DS207: Consider shorter variations of null checks + * Full docs: https://github.com/decaffeinate/decaffeinate/blob/master/docs/suggestions.md + */ +let SpellingAPIManager +const ASpell = require('./ASpell') +const LearnedWordsManager = require('./LearnedWordsManager') + +module.exports = SpellingAPIManager = { + whitelist: ['ShareLaTeX', 'sharelatex', 'LaTeX', 'http', 'https', 'www'], + + runRequest(token, request, callback) { + if (callback == null) { + callback = () => {} + } + if (request.words == null) { + return callback(new Error('malformed JSON')) + } + + const lang = request.language || 'en' + + const check = (words, callback) => + ASpell.checkWords(lang, words, (error, misspellings) => + callback(error, { misspellings }) + ) + const wordsToCheck = request.words || [] + + if (token != null) { + return LearnedWordsManager.getLearnedWords(token, function( + error, + learnedWords + ) { + if (error != null) { + return callback(error) + } + const words = wordsToCheck.slice(0, 10000) + return check(words, function(error, result) { + if (error != null) { + return callback(error) + } + result.misspellings = result.misspellings.filter(function(m) { + const word = words[m.index] + return ( + learnedWords.indexOf(word) === -1 && + SpellingAPIManager.whitelist.indexOf(word) === -1 + ) + }) + return callback(error, result) + }) + }) + } else { + return check(wordsToCheck, callback) + } + }, + + learnWord(token, request, callback) { + if (callback == null) { + callback = () => {} + } + if (request.word == null) { + return callback(new Error('malformed JSON')) + } + if (token == null) { + return callback(new Error('no token provided')) + } + + return LearnedWordsManager.learnWord(token, request.word, callback) + }, + + deleteDic(token, callback) { + return LearnedWordsManager.deleteUsersLearnedWords(token, callback) + }, + + getDic(token, callback) { + return LearnedWordsManager.getLearnedWords(token, callback) + } +} diff --git a/services/spelling/buildscript.txt b/services/spelling/buildscript.txt index 3d9abf5c04..d285c7ee17 100644 --- a/services/spelling/buildscript.txt +++ b/services/spelling/buildscript.txt @@ -1,8 +1,8 @@ spelling ---language=coffeescript --node-version=6.16.0 ---acceptance-creds=None ---dependencies=mongo,redis ---docker-repos=gcr.io/overleaf-ops ---build-target=docker --script-version=1.1.21 +--build-target=docker +--dependencies=mongo,redis +--language=es +--docker-repos=gcr.io/overleaf-ops +--acceptance-creds=None diff --git a/services/spelling/config/settings.defaults.coffee b/services/spelling/config/settings.defaults.coffee deleted file mode 100644 index 5309e8679b..0000000000 --- a/services/spelling/config/settings.defaults.coffee +++ /dev/null @@ -1,18 +0,0 @@ -Path = require('path') - -module.exports = Settings = - internal: - spelling: - port: 3005 - host: process.env["LISTEN_ADDRESS"] or "localhost" - - mongo: - url: process.env['MONGO_CONNECTION_STRING'] or "mongodb://#{process.env["MONGO_HOST"] or "localhost"}/sharelatex" - - cacheDir: Path.resolve "cache" - - - healthCheckUserId: "53c64d2fd68c8d000010bb5f" - - sentry: - dsn: process.env.SENTRY_DSN \ No newline at end of file diff --git a/services/spelling/config/settings.defaults.js b/services/spelling/config/settings.defaults.js new file mode 100644 index 0000000000..2f3bc541c3 --- /dev/null +++ b/services/spelling/config/settings.defaults.js @@ -0,0 +1,24 @@ +const Path = require('path') + +module.exports = { + internal: { + spelling: { + port: 3005, + host: process.env['LISTEN_ADDRESS'] || 'localhost' + } + }, + + mongo: { + url: + process.env['MONGO_CONNECTION_STRING'] || + `mongodb://${process.env['MONGO_HOST'] || 'localhost'}/sharelatex` + }, + + cacheDir: Path.resolve('cache'), + + healthCheckUserId: '53c64d2fd68c8d000010bb5f', + + sentry: { + dsn: process.env.SENTRY_DSN + } +} diff --git a/services/spelling/decaffeinate.sh b/services/spelling/decaffeinate.sh new file mode 100644 index 0000000000..3fbd93190b --- /dev/null +++ b/services/spelling/decaffeinate.sh @@ -0,0 +1,79 @@ +#!/usr/bin/env bash +set -ex + +# Check .eslintrc and .prettierc are present + +npm install --save-dev eslint eslint-config-prettier eslint-config-standard \ + eslint-plugin-chai-expect eslint-plugin-chai-friendly eslint-plugin-import \ + eslint-plugin-mocha eslint-plugin-node eslint-plugin-prettier \ + eslint-plugin-promise eslint-plugin-standard prettier-eslint-cli + +git add . +git commit -m "Decaffeinate: add eslint and prettier rc files" + + +echo "------------------------" +echo "----------APP-----------" +echo "------------------------" + +# bulk-decaffeinate will commit for you +npx bulk-decaffeinate convert --dir app/coffee +npx bulk-decaffeinate clean + +git mv app/coffee app/js +git commit -m "Rename app/coffee dir to app/js" + +npx prettier-eslint 'app/js/**/*.js' --write +git add . +git commit -m "Prettier: convert app/js decaffeinated files to Prettier format" + + +echo "-------------------------" +echo "--------UNIT TESTS-------" +echo "-------------------------" +npx bulk-decaffeinate convert --dir test/unit/coffee +npx bulk-decaffeinate clean + +git mv test/unit/coffee test/unit/js +git commit -m "Rename test/unit/coffee to test/unit/js" + +npx prettier-eslint 'test/unit/js/**/*.js' --write +git add . +git commit -m "Prettier: convert test/unit decaffeinated files to Prettier format" + + +echo "-------------------------" +echo "-------STRESS TESTS------" +echo "-------------------------" + +npx bulk-decaffeinate convert --dir test/stress/coffee +npx bulk-decaffeinate clean + +git mv test/stress/coffee test/stress/js +git commit -m "Rename test/stress/coffee to test/stress/js" + +npx prettier-eslint 'test/stress/js/**/*.js' --write +git add . +git commit -m "Prettier: convert test/stress decaffeinated files to Prettier format" + + +echo "--------------------------" +echo "-----INDIVIDUAL FILES-----" +echo "--------------------------" + +rm -f app.js config/settings.defaults.js +git mv app.coffee app.js +git mv config/settings.defaults.coffee config/settings.defaults.js +git commit -m "Rename individual coffee files to js files" + +decaffeinate app.js +decaffeinate config/settings.defaults.js +git add . +git commit -m "Decaffeinate: convert individual files to js" + +npx prettier-eslint 'app.js' 'config/settings.defaults.js' --write + +git add . +git commit -m "Prettier: convert individual decaffeinated files to Prettier format" + +echo "done." \ No newline at end of file diff --git a/services/spelling/nodemon.json b/services/spelling/nodemon.json index 98db38d71b..5826281b84 100644 --- a/services/spelling/nodemon.json +++ b/services/spelling/nodemon.json @@ -10,10 +10,9 @@ }, "watch": [ - "app/coffee/", - "app.coffee", + "app/js/", + "app.js", "config/" ], - "ext": "coffee" - + "ext": "js" } diff --git a/services/spelling/npm-shrinkwrap.json b/services/spelling/npm-shrinkwrap.json index a21f5efd9e..4ab45153b0 100644 --- a/services/spelling/npm-shrinkwrap.json +++ b/services/spelling/npm-shrinkwrap.json @@ -1849,4 +1849,4 @@ "resolved": "https://registry.npmjs.org/yallist/-/yallist-2.1.2.tgz" } } -} +} \ No newline at end of file diff --git a/services/spelling/package.json b/services/spelling/package.json index b06c753c4e..70acefef16 100644 --- a/services/spelling/package.json +++ b/services/spelling/package.json @@ -8,16 +8,19 @@ }, "scripts": { "compile:app": "([ -e app/coffee ] && coffee -m $COFFEE_OPTIONS -o app/js -c app/coffee || echo 'No CoffeeScript folder to compile') && ( [ -e app.coffee ] && coffee -m $COFFEE_OPTIONS -c app.coffee || echo 'No CoffeeScript app to compile')", - "start": "npm run compile:app && node $NODE_APP_OPTIONS app.js", - "test:acceptance:_run": "mocha --recursive --reporter spec --timeout 30000 --exit $@ test/acceptance/js", - "test:acceptance": "npm run compile:app && npm run compile:acceptance_tests && npm run test:acceptance:_run -- --grep=$MOCHA_GREP", + "start": "node $NODE_APP_OPTIONS app.js", + "test:acceptance:_run": "mocha --recursive --reporter spec --timeout 15000 --exit $@ test/acceptance/js", + "test:acceptance": "npm run test:acceptance:_run -- --grep=$MOCHA_GREP", "test:unit:_run": "mocha --recursive --reporter spec --exit $@ test/unit/js", - "test:unit": "npm run compile:app && npm run compile:unit_tests && npm run test:unit:_run -- --grep=$MOCHA_GREP", + "test:unit": "npm run test:unit:_run -- --grep=$MOCHA_GREP", "compile:unit_tests": "[ ! -e test/unit/coffee ] && echo 'No unit tests to compile' || coffee -o test/unit/js -c test/unit/coffee", "compile:acceptance_tests": "[ ! -e test/acceptance/coffee ] && echo 'No acceptance tests to compile' || coffee -o test/acceptance/js -c test/acceptance/coffee", "compile:all": "npm run compile:app && npm run compile:unit_tests && npm run compile:acceptance_tests && npm run compile:smoke_tests", "nodemon": "nodemon --config nodemon.json", - "compile:smoke_tests": "[ ! -e test/smoke/coffee ] && echo 'No smoke tests to compile' || coffee -o test/smoke/js -c test/smoke/coffee" + "compile:smoke_tests": "[ ! -e test/smoke/coffee ] && echo 'No smoke tests to compile' || coffee -o test/smoke/js -c test/smoke/coffee", + "lint": "node_modules/.bin/eslint .", + "format": "node_modules/.bin/prettier-eslint '**/*.js' --list-different", + "format:fix": "node_modules/.bin/prettier-eslint '**/*.js' --write" }, "version": "0.1.4", "dependencies": { @@ -39,7 +42,19 @@ "devDependencies": { "bunyan": "^1.0.0", "chai": "", + "eslint": "^6.0.1", + "eslint-config-prettier": "^6.0.0", + "eslint-config-standard": "^12.0.0", + "eslint-plugin-chai-expect": "^2.0.1", + "eslint-plugin-chai-friendly": "^0.4.1", + "eslint-plugin-import": "^2.18.0", + "eslint-plugin-mocha": "^5.3.0", + "eslint-plugin-node": "^9.1.0", + "eslint-plugin-prettier": "^3.1.0", + "eslint-plugin-promise": "^4.2.1", + "eslint-plugin-standard": "^4.0.0", "mocha": "^4.1.0", + "prettier-eslint-cli": "^5.0.0", "sandboxed-module": "", "sinon": "^1.17.0" } diff --git a/services/spelling/test/stress/coffee/stressTest.coffee b/services/spelling/test/stress/coffee/stressTest.coffee deleted file mode 100644 index 5ff53996c1..0000000000 --- a/services/spelling/test/stress/coffee/stressTest.coffee +++ /dev/null @@ -1,86 +0,0 @@ -# N requests in parallel -# send P correct words and Q incorrect words -# generate incorrect words by qq+random - -async = require "async" -request = require "request" -fs = require "fs" - -# created with -# aspell -d en dump master | aspell -l en expand | shuf -n 150000 > words.txt -WORDS = "words.txt" -wordlist = fs.readFileSync(WORDS).toString().split('\n').filter (w) -> - w.match(/^[a-z]+$/) - -generateCorrectWords = (n) -> - words = [] - N = if Math.random() > 0.5 then wordlist.length else 10 - for i in [1 .. n] - j = Math.floor(N * Math.random()) - words.push wordlist[j] - return words - -generateIncorrectWords = (n) -> - words = [] - N = wordlist.length - for i in [1 .. n] - j = Math.floor(N * Math.random()) - words.push("qzxq" + wordlist[j]) - return words - -make_request = (correctWords, incorrectWords, callback) -> - correctSet = generateCorrectWords(correctWords) - incorrectSet = generateIncorrectWords(incorrectWords) - correctSet.push('constructor') - incorrectSet.push('qzxqfoofoofoo') - full = correctSet.concat incorrectSet - bad = [] - for w, i in correctSet - bad[i] = false - for w, i in incorrectSet - bad[i+correctSet.length] = true - k = full.length - full.forEach (e, i) -> - j = Math.floor(k * Math.random()) - [ full[i], full[j] ] = [ full[j], full[i] ] - [ bad[i], bad[j] ] = [ bad[j], bad[i] ] - expected = [] - for tf, i in bad - if tf - expected.push {index: i, word: full[i]} - request.post 'http://localhost:3005/user/1/check', json:true, body: {words: full}, (err, req, body) -> - misspellings = body.misspellings - console.log JSON.stringify({full: full, misspellings: misspellings}) - if expected.length != misspellings.length - console.log "ERROR: length mismatch", expected.length, misspellings.length - console.log full, bad - console.log 'expected', expected, 'mispellings', misspellings - for i in [0 .. Math.max(expected.length, misspellings.length)-1] - if expected[i].index != misspellings[i].index - console.log "ERROR", i, expected[i], misspellings[i], full[misspellings[i].index] - for m in misspellings - console.log full[m.index], "=>", m - process.exit() - callback("error") - else - for m,i in body.misspellings - if m.index != expected[i].index - console.log "ERROR AT RESULT", i, m, expected[i] - process.exit() - callback("error") - callback(null, full) - -q = async.queue (task, callback) -> - setTimeout () -> - make_request task.correct, task.incorrect, callback - , Math.random() * 100 -, 3 - -q.drain = () -> - console.log('all items have been processed'); - -for i in [0 .. 1000] - q.push({correct: Math.floor(30*Math.random()) + 1, incorrect: Math.floor(3*Math.random())}) - # if Math.random() < 0.1 - # else - # q.push({correct: Math.floor(100*Math.random()) + 1, incorrect: Math.floor(3*Math.random())}) diff --git a/services/spelling/test/stress/js/stressTest.js b/services/spelling/test/stress/js/stressTest.js new file mode 100644 index 0000000000..4a4d09568e --- /dev/null +++ b/services/spelling/test/stress/js/stressTest.js @@ -0,0 +1,162 @@ +/* eslint-disable */ +// TODO: This file was created by bulk-decaffeinate. +// Sanity-check the conversion and remove this comment. +/* + * decaffeinate suggestions: + * DS101: Remove unnecessary use of Array.from + * DS102: Remove unnecessary code created because of implicit returns + * DS104: Avoid inline assignments + * DS202: Simplify dynamic range loops + * Full docs: https://github.com/decaffeinate/decaffeinate/blob/master/docs/suggestions.md + */ +// N requests in parallel +// send P correct words and Q incorrect words +// generate incorrect words by qq+random + +const async = require('async') +const request = require('request') +const fs = require('fs') + +// created with +// aspell -d en dump master | aspell -l en expand | shuf -n 150000 > words.txt +const WORDS = 'words.txt' +const wordlist = fs + .readFileSync(WORDS) + .toString() + .split('\n') + .filter(w => w.match(/^[a-z]+$/)) + +const generateCorrectWords = function(n) { + const words = [] + const N = Math.random() > 0.5 ? wordlist.length : 10 + for ( + let i = 1, end = n, asc = 1 <= end; + asc ? i <= end : i >= end; + asc ? i++ : i-- + ) { + const j = Math.floor(N * Math.random()) + words.push(wordlist[j]) + } + return words +} + +const generateIncorrectWords = function(n) { + const words = [] + const N = wordlist.length + for ( + let i = 1, end = n, asc = 1 <= end; + asc ? i <= end : i >= end; + asc ? i++ : i-- + ) { + const j = Math.floor(N * Math.random()) + words.push(`qzxq${wordlist[j]}`) + } + return words +} + +const make_request = function(correctWords, incorrectWords, callback) { + let i, j, w + let i1 + let j1 + const correctSet = generateCorrectWords(correctWords) + const incorrectSet = generateIncorrectWords(incorrectWords) + correctSet.push('constructor') + incorrectSet.push('qzxqfoofoofoo') + const full = correctSet.concat(incorrectSet) + const bad = [] + for (j = 0, i = j; j < correctSet.length; j++, i = j) { + w = correctSet[i] + bad[i] = false + } + for (i1 = 0, i = i1; i1 < incorrectSet.length; i1++, i = i1) { + w = incorrectSet[i] + bad[i + correctSet.length] = true + } + const k = full.length + full.forEach(function(e, i) { + let ref + j = Math.floor(k * Math.random()) + ;[full[i], full[j]] = Array.from([full[j], full[i]]) + return ([bad[i], bad[j]] = Array.from((ref = [bad[j], bad[i]]))), ref + }) + const expected = [] + for (j1 = 0, i = j1; j1 < bad.length; j1++, i = j1) { + const tf = bad[i] + if (tf) { + expected.push({ index: i, word: full[i] }) + } + } + return request.post( + 'http://localhost:3005/user/1/check', + { json: true, body: { words: full } }, + function(err, req, body) { + let m + const { misspellings } = body + console.log(JSON.stringify({ full, misspellings })) + if (expected.length !== misspellings.length) { + let asc, end + console.log( + 'ERROR: length mismatch', + expected.length, + misspellings.length + ) + console.log(full, bad) + console.log('expected', expected, 'mispellings', misspellings) + for ( + i = 0, + end = Math.max(expected.length, misspellings.length) - 1, + asc = 0 <= end; + asc ? i <= end : i >= end; + asc ? i++ : i-- + ) { + if (expected[i].index !== misspellings[i].index) { + console.log( + 'ERROR', + i, + expected[i], + misspellings[i], + full[misspellings[i].index] + ) + } + } + for (m of Array.from(misspellings)) { + console.log(full[m.index], '=>', m) + } + process.exit() + callback('error') + } else { + for (i = 0; i < body.misspellings.length; i++) { + m = body.misspellings[i] + if (m.index !== expected[i].index) { + console.log('ERROR AT RESULT', i, m, expected[i]) + process.exit() + callback('error') + } + } + } + return callback(null, full) + } + ) +} + +const q = async.queue( + (task, callback) => + setTimeout( + () => make_request(task.correct, task.incorrect, callback), + Math.random() * 100 + ), + + 3 +) + +q.drain = () => console.log('all items have been processed') + +for (let i = 0; i <= 1000; i++) { + q.push({ + correct: Math.floor(30 * Math.random()) + 1, + incorrect: Math.floor(3 * Math.random()) + }) +} +// if Math.random() < 0.1 +// else +// q.push({correct: Math.floor(100*Math.random()) + 1, incorrect: Math.floor(3*Math.random())}) diff --git a/services/spelling/test/stress/coffee/words.txt b/services/spelling/test/stress/js/words.txt similarity index 100% rename from services/spelling/test/stress/coffee/words.txt rename to services/spelling/test/stress/js/words.txt diff --git a/services/spelling/test/unit/coffee/ASpellTests.coffee b/services/spelling/test/unit/coffee/ASpellTests.coffee deleted file mode 100644 index e4adae0a8a..0000000000 --- a/services/spelling/test/unit/coffee/ASpellTests.coffee +++ /dev/null @@ -1,70 +0,0 @@ -sinon = require 'sinon' -chai = require 'chai' -should = chai.should() -SandboxedModule = require('sandboxed-module') -assert = require("chai").assert - -describe "ASpell", -> - beforeEach -> - @ASpell = SandboxedModule.require "../../../app/js/ASpell", requires: - "logger-sharelatex": - log:-> - info:-> - err:-> - 'metrics-sharelatex': - gauge:-> - inc: -> - - describe "a correctly spelled word", -> - beforeEach (done) -> - @ASpell.checkWords "en", ["word"], (error, @result) => done() - - it "should not correct the word", -> - @result.length.should.equal 0 - - describe "a misspelled word", -> - beforeEach (done) -> - @ASpell.checkWords "en", ["bussines"], (error, @result) => done() - - it "should correct the word", -> - @result.length.should.equal 1 - @result[0].suggestions.indexOf("business").should.not.equal -1 - - describe "multiple words", -> - beforeEach (done) -> - @ASpell.checkWords "en", ["bussines", "word", "neccesary"], (error, @result) => done() - - it "should correct the incorrect words", -> - @result[0].index.should.equal 0 - @result[0].suggestions.indexOf("business").should.not.equal -1 - @result[1].index.should.equal 2 - @result[1].suggestions.indexOf("necessary").should.not.equal -1 - - describe "without a valid language", -> - beforeEach (done) -> - @ASpell.checkWords "notALang", ["banana"], (@error, @result) => done() - - it "should return an error", -> - should.exist @error - - describe "when there are no suggestions", -> - beforeEach (done) -> - @ASpell.checkWords "en", ["asdkfjalkdjfadhfkajsdhfashdfjhadflkjadhflajsd"], (@error, @result) => done() - - it "should return a blank array", -> - @result.length.should.equal 1 - assert.deepEqual @result[0].suggestions, [] - - describe "when the request times out", -> - beforeEach (done) -> - words = ("abcdefg" for i in [0..1000]) - @ASpell.ASPELL_TIMEOUT = 1 - @start = Date.now() - @ASpell.checkWords "en", words, (error, @result) => done() - - # Note that this test fails on OS X, due to differing pipe behaviour - # on killing the child process. It can be tested successfully on Travis - # or the CI server. - it "should return in reasonable time", () -> - delta = Date.now()-@start - delta.should.be.below(@ASpell.ASPELL_TIMEOUT + 1000) diff --git a/services/spelling/test/unit/coffee/LearnedWordsManagerTests.coffee b/services/spelling/test/unit/coffee/LearnedWordsManagerTests.coffee deleted file mode 100644 index 3819a2fdd3..0000000000 --- a/services/spelling/test/unit/coffee/LearnedWordsManagerTests.coffee +++ /dev/null @@ -1,97 +0,0 @@ -sinon = require('sinon') -chai = require 'chai' -expect = chai.expect -SandboxedModule = require('sandboxed-module') -modulePath = require('path').join __dirname, '../../../app/js/LearnedWordsManager' -assert = require("chai").assert -should = require("chai").should() -describe "LearnedWordsManager", -> - beforeEach -> - @token = "a6b3cd919ge" - @callback = sinon.stub() - @db = - spellingPreferences: - update: sinon.stub().callsArg(3) - @cache = - get:sinon.stub() - set:sinon.stub() - del:sinon.stub() - @LearnedWordsManager = SandboxedModule.require modulePath, requires: - "./DB" : @db - "./MongoCache":@cache - "logger-sharelatex": - log:-> - err:-> - info:-> - 'metrics-sharelatex': {timeAsyncMethod: sinon.stub(), inc: sinon.stub()} - - describe "learnWord", -> - beforeEach -> - @word = "instanton" - @LearnedWordsManager.learnWord @token, @word, @callback - - it "should insert the word in the word list in the database", -> - expect( - @db.spellingPreferences.update.calledWith({ - token: @token - }, { - $push : learnedWords: @word - }, { - upsert: true - }) - ).to.equal true - - it "should call the callback", -> - expect(@callback.called).to.equal true - - describe "getLearnedWords", -> - beforeEach -> - @wordList = ["apples", "bananas", "pears"] - @db.spellingPreferences.findOne = (conditions, callback) => - callback null, learnedWords: @wordList - sinon.spy @db.spellingPreferences, "findOne" - @LearnedWordsManager.getLearnedWords @token, @callback - - it "should get the word list for the given user", -> - expect( - @db.spellingPreferences.findOne.calledWith token: @token - ).to.equal true - - it "should return the word list in the callback", -> - expect(@callback.calledWith null, @wordList).to.equal true - - - describe "caching the result", -> - it 'should use the cache first if it is primed', (done)-> - @wordList = ["apples", "bananas", "pears"] - @cache.get.returns(@wordList) - @db.spellingPreferences.findOne = sinon.stub() - @LearnedWordsManager.getLearnedWords @token, (err, spellings)=> - @db.spellingPreferences.findOne.called.should.equal false - assert.deepEqual @wordList, spellings - done() - - it 'should set the cache after hitting the db', (done)-> - @wordList = ["apples", "bananas", "pears"] - @db.spellingPreferences.findOne = sinon.stub().callsArgWith(1, null, learnedWords: @wordList) - @LearnedWordsManager.getLearnedWords @token, (err, spellings)=> - @cache.set.calledWith(@token, @wordList).should.equal true - done() - - it 'should break cache when update is called', (done)-> - @word = "instanton" - @LearnedWordsManager.learnWord @token, @word, => - @cache.del.calledWith(@token).should.equal true - done() - - - describe "deleteUsersLearnedWords", -> - beforeEach -> - @db.spellingPreferences.remove = sinon.stub().callsArgWith(1) - - - it "should get the word list for the given user", (done)-> - @LearnedWordsManager.deleteUsersLearnedWords @token, => - @db.spellingPreferences.remove.calledWith(token: @token).should.equal true - done() - diff --git a/services/spelling/test/unit/coffee/SpellingAPIManagerTests.coffee b/services/spelling/test/unit/coffee/SpellingAPIManagerTests.coffee deleted file mode 100644 index 83e117f252..0000000000 --- a/services/spelling/test/unit/coffee/SpellingAPIManagerTests.coffee +++ /dev/null @@ -1,119 +0,0 @@ -sinon = require('sinon') -chai = require 'chai' -expect = chai.expect -chai.should() -SandboxedModule = require('sandboxed-module') -modulePath = require('path').join __dirname, '../../../app/js/SpellingAPIManager' - -describe "SpellingAPIManager", -> - beforeEach -> - @token = "user-id-123" - @ASpell = {} - @learnedWords = ["lerned"] - @LearnedWordsManager = - getLearnedWords: sinon.stub().callsArgWith(1, null, @learnedWords) - learnWord: sinon.stub().callsArg(2) - - @SpellingAPIManager = SandboxedModule.require modulePath, requires: - "./ASpell" : @ASpell - "./LearnedWordsManager" : @LearnedWordsManager - - describe "runRequest", -> - beforeEach -> - @nonLearnedWords = ["some", "words", "htat", "are", "speled", "rong", "lerned"] - @allWords = @nonLearnedWords.concat(@learnedWords) - @misspellings = [ - { index: 2, suggestions: ["that"] } - { index: 4, suggestions: ["spelled"] } - { index: 5, suggestions: ["wrong", "ring"] } - { index: 6, suggestions: ["learned"] } - ] - @misspellingsWithoutLearnedWords = @misspellings.slice(0,3) - - @ASpell.checkWords = (lang, word, callback) => - callback null, @misspellings - sinon.spy @ASpell, "checkWords" - - describe "with sensible JSON", -> - beforeEach (done) -> - @SpellingAPIManager.runRequest @token, words: @allWords, (error, @result) => done() - - it "should return the words that are spelled incorrectly and not learned", -> - expect(@result.misspellings).to.deep.equal @misspellingsWithoutLearnedWords - - describe "with a missing words array", -> - beforeEach (done) -> - @SpellingAPIManager.runRequest @token, {}, (@error, @result) => done() - - it "should return an error", -> - expect(@error).to.exist - expect(@error).to.be.instanceof Error - expect(@error.message).to.equal "malformed JSON" - - describe "with a missing token", -> - beforeEach (done) -> - @SpellingAPIManager.runRequest null, words: @allWords, (@error, @result) => done() - - it "should spell check without using any learned words", -> - @LearnedWordsManager.getLearnedWords.called.should.equal false - - describe "without a language", -> - beforeEach (done) -> - @SpellingAPIManager.runRequest @token, words: @allWords, (error, @result) => done() - - it "should use en as the default", -> - @ASpell.checkWords.calledWith("en").should.equal true - - describe "with a language", -> - beforeEach (done) -> - @SpellingAPIManager.runRequest @token, { - words: @allWords - language: @language = "fr" - }, (error, @result) => done() - - it "should use the language", -> - @ASpell.checkWords.calledWith(@language).should.equal true - - describe "with a very large collection of words", -> - beforeEach (done) -> - @manyWords = ("word" for i in [1..100000]) - @SpellingAPIManager.runRequest @token, words: @manyWords, (error, @result) => done() - - it "should truncate to 10,000 words", -> - @ASpell.checkWords.calledWith(sinon.match.any, @manyWords.slice(0, 10000)).should.equal true - - describe 'with words from the whitelist', -> - beforeEach (done) -> - @whitelistWord = @SpellingAPIManager.whitelist[0] - @words = ["One", "Two", @whitelistWord] - @SpellingAPIManager.runRequest @token, words: @words, (error, @result) => done() - - it 'should ignore the white-listed word', -> - expect(@result.misspellings.length).to.equal @misspellings.length-1 - - describe "learnWord", -> - describe "without a token", -> - beforeEach (done) -> @SpellingAPIManager.learnWord null, word: "banana", (@error) => done() - - it "should return an error", -> - expect(@error).to.exist - expect(@error).to.be.instanceof Error - expect(@error.message).to.equal "no token provided" - - describe "without a word", -> - beforeEach (done) -> @SpellingAPIManager.learnWord @token, {}, (@error) => done() - - it "should return an error", -> - expect(@error).to.exist - expect(@error).to.be.instanceof Error - expect(@error.message).to.equal "malformed JSON" - - describe "with a word and a token", -> - beforeEach (done) -> - @word = "banana" - @SpellingAPIManager.learnWord @token, word: @word, (@error) => done() - - it "should call LearnedWordsManager.learnWord", -> - @LearnedWordsManager.learnWord.calledWith(@token, @word).should.equal true - - diff --git a/services/spelling/test/unit/js/ASpellTests.js b/services/spelling/test/unit/js/ASpellTests.js new file mode 100644 index 0000000000..98a3b381de --- /dev/null +++ b/services/spelling/test/unit/js/ASpellTests.js @@ -0,0 +1,145 @@ +/* eslint-disable + handle-callback-err, + no-undef +*/ +// TODO: This file was created by bulk-decaffeinate. +// Sanity-check the conversion and remove this comment. +/* + * decaffeinate suggestions: + * DS102: Remove unnecessary code created because of implicit returns + * Full docs: https://github.com/decaffeinate/decaffeinate/blob/master/docs/suggestions.md + */ +const chai = require('chai') +const should = chai.should() +const SandboxedModule = require('sandboxed-module') +const { assert } = require('chai') + +describe('ASpell', function() { + beforeEach(function() { + return (this.ASpell = SandboxedModule.require('../../../app/js/ASpell', { + requires: { + 'logger-sharelatex': { + log() {}, + info() {}, + err() {} + }, + 'metrics-sharelatex': { + gauge() {}, + inc() {} + } + } + })) + }) + + describe('a correctly spelled word', function() { + beforeEach(function(done) { + return this.ASpell.checkWords('en', ['word'], (error, result) => { + this.result = result + return done() + }) + }) + + return it('should not correct the word', function() { + return this.result.length.should.equal(0) + }) + }) + + describe('a misspelled word', function() { + beforeEach(function(done) { + return this.ASpell.checkWords('en', ['bussines'], (error, result) => { + this.result = result + return done() + }) + }) + + return it('should correct the word', function() { + this.result.length.should.equal(1) + return this.result[0].suggestions.indexOf('business').should.not.equal(-1) + }) + }) + + describe('multiple words', function() { + beforeEach(function(done) { + return this.ASpell.checkWords( + 'en', + ['bussines', 'word', 'neccesary'], + (error, result) => { + this.result = result + return done() + } + ) + }) + + return it('should correct the incorrect words', function() { + this.result[0].index.should.equal(0) + this.result[0].suggestions.indexOf('business').should.not.equal(-1) + this.result[1].index.should.equal(2) + return this.result[1].suggestions + .indexOf('necessary') + .should.not.equal(-1) + }) + }) + + describe('without a valid language', function() { + beforeEach(function(done) { + return this.ASpell.checkWords('notALang', ['banana'], (error, result) => { + this.error = error + this.result = result + return done() + }) + }) + + return it('should return an error', function() { + return should.exist(this.error) + }) + }) + + describe('when there are no suggestions', function() { + beforeEach(function(done) { + return this.ASpell.checkWords( + 'en', + ['asdkfjalkdjfadhfkajsdhfashdfjhadflkjadhflajsd'], + (error, result) => { + this.error = error + this.result = result + return done() + } + ) + }) + + return it('should return a blank array', function() { + this.result.length.should.equal(1) + return assert.deepEqual(this.result[0].suggestions, []) + }) + }) + + return describe('when the request times out', function() { + beforeEach(function(done) { + const words = __range__(0, 1000, true).map(i => 'abcdefg') + this.ASpell.ASPELL_TIMEOUT = 1 + this.start = Date.now() + return this.ASpell.checkWords('en', words, (error, result) => { + this.result = result + return done() + }) + }) + + // Note that this test fails on OS X, due to differing pipe behaviour + // on killing the child process. It can be tested successfully on Travis + // or the CI server. + return it('should return in reasonable time', function() { + const delta = Date.now() - this.start + return delta.should.be.below(this.ASpell.ASPELL_TIMEOUT + 1000) + }) + }) +}) + +function __range__(left, right, inclusive) { + let range = [] + let ascending = left < right + let end = !inclusive ? right : ascending ? right + 1 : right - 1 + for (let i = left; ascending ? i < end : i > end; ascending ? i++ : i--) { + range.push(i) + } + return range +} diff --git a/services/spelling/test/unit/js/LearnedWordsManagerTests.js b/services/spelling/test/unit/js/LearnedWordsManagerTests.js new file mode 100644 index 0000000000..76122b1d5c --- /dev/null +++ b/services/spelling/test/unit/js/LearnedWordsManagerTests.js @@ -0,0 +1,163 @@ +/* eslint-disable + handle-callback-err, + no-undef +*/ +// TODO: This file was created by bulk-decaffeinate. +// Sanity-check the conversion and remove this comment. +/* + * decaffeinate suggestions: + * DS102: Remove unnecessary code created because of implicit returns + * Full docs: https://github.com/decaffeinate/decaffeinate/blob/master/docs/suggestions.md + */ +const sinon = require('sinon') +const chai = require('chai') +const { expect } = chai +const SandboxedModule = require('sandboxed-module') +const modulePath = require('path').join( + __dirname, + '../../../app/js/LearnedWordsManager' +) +const { assert } = require('chai') +describe('LearnedWordsManager', function() { + beforeEach(function() { + this.token = 'a6b3cd919ge' + this.callback = sinon.stub() + this.db = { + spellingPreferences: { + update: sinon.stub().callsArg(3) + } + } + this.cache = { + get: sinon.stub(), + set: sinon.stub(), + del: sinon.stub() + } + return (this.LearnedWordsManager = SandboxedModule.require(modulePath, { + requires: { + './DB': this.db, + './MongoCache': this.cache, + 'logger-sharelatex': { + log() {}, + err() {}, + info() {} + }, + 'metrics-sharelatex': { + timeAsyncMethod: sinon.stub(), + inc: sinon.stub() + } + } + })) + }) + + describe('learnWord', function() { + beforeEach(function() { + this.word = 'instanton' + return this.LearnedWordsManager.learnWord( + this.token, + this.word, + this.callback + ) + }) + + it('should insert the word in the word list in the database', function() { + return expect( + this.db.spellingPreferences.update.calledWith( + { + token: this.token + }, + { + $push: { learnedWords: this.word } + }, + { + upsert: true + } + ) + ).to.equal(true) + }) + + return it('should call the callback', function() { + return expect(this.callback.called).to.equal(true) + }) + }) + + describe('getLearnedWords', function() { + beforeEach(function() { + this.wordList = ['apples', 'bananas', 'pears'] + this.db.spellingPreferences.findOne = (conditions, callback) => { + return callback(null, { learnedWords: this.wordList }) + } + sinon.spy(this.db.spellingPreferences, 'findOne') + return this.LearnedWordsManager.getLearnedWords(this.token, this.callback) + }) + + it('should get the word list for the given user', function() { + return expect( + this.db.spellingPreferences.findOne.calledWith({ token: this.token }) + ).to.equal(true) + }) + + return it('should return the word list in the callback', function() { + return expect(this.callback.calledWith(null, this.wordList)).to.equal( + true + ) + }) + }) + + describe('caching the result', function() { + it('should use the cache first if it is primed', function(done) { + this.wordList = ['apples', 'bananas', 'pears'] + this.cache.get.returns(this.wordList) + this.db.spellingPreferences.findOne = sinon.stub() + return this.LearnedWordsManager.getLearnedWords( + this.token, + (err, spellings) => { + this.db.spellingPreferences.findOne.called.should.equal(false) + assert.deepEqual(this.wordList, spellings) + return done() + } + ) + }) + + it('should set the cache after hitting the db', function(done) { + this.wordList = ['apples', 'bananas', 'pears'] + this.db.spellingPreferences.findOne = sinon + .stub() + .callsArgWith(1, null, { learnedWords: this.wordList }) + return this.LearnedWordsManager.getLearnedWords( + this.token, + (err, spellings) => { + this.cache.set + .calledWith(this.token, this.wordList) + .should.equal(true) + return done() + } + ) + }) + + return it('should break cache when update is called', function(done) { + this.word = 'instanton' + return this.LearnedWordsManager.learnWord(this.token, this.word, () => { + this.cache.del.calledWith(this.token).should.equal(true) + return done() + }) + }) + }) + + return describe('deleteUsersLearnedWords', function() { + beforeEach(function() { + return (this.db.spellingPreferences.remove = sinon.stub().callsArgWith(1)) + }) + + return it('should get the word list for the given user', function(done) { + return this.LearnedWordsManager.deleteUsersLearnedWords( + this.token, + () => { + this.db.spellingPreferences.remove + .calledWith({ token: this.token }) + .should.equal(true) + return done() + } + ) + }) + }) +}) diff --git a/services/spelling/test/unit/js/SpellingAPIManagerTests.js b/services/spelling/test/unit/js/SpellingAPIManagerTests.js new file mode 100644 index 0000000000..079b84bb06 --- /dev/null +++ b/services/spelling/test/unit/js/SpellingAPIManagerTests.js @@ -0,0 +1,271 @@ +/* eslint-disable + handle-callback-err, + no-undef +*/ +// TODO: This file was created by bulk-decaffeinate. +// Sanity-check the conversion and remove this comment. +/* + * decaffeinate suggestions: + * DS102: Remove unnecessary code created because of implicit returns + * Full docs: https://github.com/decaffeinate/decaffeinate/blob/master/docs/suggestions.md + */ +const sinon = require('sinon') +const chai = require('chai') +const { expect } = chai +chai.should() +const SandboxedModule = require('sandboxed-module') +const modulePath = require('path').join( + __dirname, + '../../../app/js/SpellingAPIManager' +) + +describe('SpellingAPIManager', function() { + beforeEach(function() { + this.token = 'user-id-123' + this.ASpell = {} + this.learnedWords = ['lerned'] + this.LearnedWordsManager = { + getLearnedWords: sinon.stub().callsArgWith(1, null, this.learnedWords), + learnWord: sinon.stub().callsArg(2) + } + + return (this.SpellingAPIManager = SandboxedModule.require(modulePath, { + requires: { + './ASpell': this.ASpell, + './LearnedWordsManager': this.LearnedWordsManager + } + })) + }) + + describe('runRequest', function() { + beforeEach(function() { + this.nonLearnedWords = [ + 'some', + 'words', + 'htat', + 'are', + 'speled', + 'rong', + 'lerned' + ] + this.allWords = this.nonLearnedWords.concat(this.learnedWords) + this.misspellings = [ + { index: 2, suggestions: ['that'] }, + { index: 4, suggestions: ['spelled'] }, + { index: 5, suggestions: ['wrong', 'ring'] }, + { index: 6, suggestions: ['learned'] } + ] + this.misspellingsWithoutLearnedWords = this.misspellings.slice(0, 3) + + this.ASpell.checkWords = (lang, word, callback) => { + return callback(null, this.misspellings) + } + return sinon.spy(this.ASpell, 'checkWords') + }) + + describe('with sensible JSON', function() { + beforeEach(function(done) { + return this.SpellingAPIManager.runRequest( + this.token, + { words: this.allWords }, + (error, result) => { + this.result = result + return done() + } + ) + }) + + return it('should return the words that are spelled incorrectly and not learned', function() { + return expect(this.result.misspellings).to.deep.equal( + this.misspellingsWithoutLearnedWords + ) + }) + }) + + describe('with a missing words array', function() { + beforeEach(function(done) { + return this.SpellingAPIManager.runRequest( + this.token, + {}, + (error, result) => { + this.error = error + this.result = result + return done() + } + ) + }) + + return it('should return an error', function() { + expect(this.error).to.exist + expect(this.error).to.be.instanceof(Error) + return expect(this.error.message).to.equal('malformed JSON') + }) + }) + + describe('with a missing token', function() { + beforeEach(function(done) { + return this.SpellingAPIManager.runRequest( + null, + { words: this.allWords }, + (error, result) => { + this.error = error + this.result = result + return done() + } + ) + }) + + return it('should spell check without using any learned words', function() { + return this.LearnedWordsManager.getLearnedWords.called.should.equal( + false + ) + }) + }) + + describe('without a language', function() { + beforeEach(function(done) { + return this.SpellingAPIManager.runRequest( + this.token, + { words: this.allWords }, + (error, result) => { + this.result = result + return done() + } + ) + }) + + return it('should use en as the default', function() { + return this.ASpell.checkWords.calledWith('en').should.equal(true) + }) + }) + + describe('with a language', function() { + beforeEach(function(done) { + return this.SpellingAPIManager.runRequest( + this.token, + { + words: this.allWords, + language: (this.language = 'fr') + }, + (error, result) => { + this.result = result + return done() + } + ) + }) + + return it('should use the language', function() { + return this.ASpell.checkWords + .calledWith(this.language) + .should.equal(true) + }) + }) + + describe('with a very large collection of words', function() { + beforeEach(function(done) { + this.manyWords = __range__(1, 100000, true).map(i => 'word') + return this.SpellingAPIManager.runRequest( + this.token, + { words: this.manyWords }, + (error, result) => { + this.result = result + return done() + } + ) + }) + + return it('should truncate to 10,000 words', function() { + return this.ASpell.checkWords + .calledWith(sinon.match.any, this.manyWords.slice(0, 10000)) + .should.equal(true) + }) + }) + + return describe('with words from the whitelist', function() { + beforeEach(function(done) { + this.whitelistWord = this.SpellingAPIManager.whitelist[0] + this.words = ['One', 'Two', this.whitelistWord] + return this.SpellingAPIManager.runRequest( + this.token, + { words: this.words }, + (error, result) => { + this.result = result + return done() + } + ) + }) + + return it('should ignore the white-listed word', function() { + return expect(this.result.misspellings.length).to.equal( + this.misspellings.length - 1 + ) + }) + }) + }) + + return describe('learnWord', function() { + describe('without a token', function() { + beforeEach(function(done) { + return this.SpellingAPIManager.learnWord( + null, + { word: 'banana' }, + error => { + this.error = error + return done() + } + ) + }) + + return it('should return an error', function() { + expect(this.error).to.exist + expect(this.error).to.be.instanceof(Error) + return expect(this.error.message).to.equal('no token provided') + }) + }) + + describe('without a word', function() { + beforeEach(function(done) { + return this.SpellingAPIManager.learnWord(this.token, {}, error => { + this.error = error + return done() + }) + }) + + return it('should return an error', function() { + expect(this.error).to.exist + expect(this.error).to.be.instanceof(Error) + return expect(this.error.message).to.equal('malformed JSON') + }) + }) + + return describe('with a word and a token', function() { + beforeEach(function(done) { + this.word = 'banana' + return this.SpellingAPIManager.learnWord( + this.token, + { word: this.word }, + error => { + this.error = error + return done() + } + ) + }) + + return it('should call LearnedWordsManager.learnWord', function() { + return this.LearnedWordsManager.learnWord + .calledWith(this.token, this.word) + .should.equal(true) + }) + }) + }) +}) + +function __range__(left, right, inclusive) { + let range = [] + let ascending = left < right + let end = !inclusive ? right : ascending ? right + 1 : right - 1 + for (let i = left; ascending ? i < end : i > end; ascending ? i++ : i--) { + range.push(i) + } + return range +}