mirror of
https://github.com/overleaf/overleaf.git
synced 2024-11-21 20:47:08 -05:00
Initial decaffeination (#24)
This commit is contained in:
Miguel Serrano
GitHub
parent
1e74d545c4
commit
0eba057cef
43 changed files with 1840 additions and 940 deletions
|
|
@ -5,5 +5,3 @@ gitrev
|
|||
.npm
|
||||
.nvmrc
|
||||
nodemon.json
|
||||
app.js
|
||||
**/js/*
|
||||
|
|
|
|||
36
services/spelling/.eslintrc
Normal file
36
services/spelling/.eslintrc
Normal file
|
|
@ -0,0 +1,36 @@
|
|||
// this file was auto-generated, do not edit it directly.
|
||||
// instead run bin/update_build_scripts from
|
||||
// https://github.com/sharelatex/sharelatex-dev-environment
|
||||
// Version: 1.1.21
|
||||
{
|
||||
"extends": [
|
||||
"standard",
|
||||
"prettier",
|
||||
"prettier/standard",
|
||||
],
|
||||
"plugins": [
|
||||
"mocha",
|
||||
"chai-expect",
|
||||
"chai-friendly"
|
||||
],
|
||||
"env": {
|
||||
"node": true
|
||||
},
|
||||
"rules": {
|
||||
// Add some mocha specific rules
|
||||
"mocha/handle-done-callback": "error",
|
||||
"mocha/no-exclusive-tests": "error",
|
||||
"mocha/no-global-tests": "error",
|
||||
"mocha/no-identical-title": "error",
|
||||
"mocha/no-nested-tests": "error",
|
||||
"mocha/no-pending-tests": "error",
|
||||
"mocha/no-skipped-tests": "error",
|
||||
|
||||
// Add some chai specific rules
|
||||
"chai-expect/missing-assertion": "error",
|
||||
"chai-expect/terminating-properties": "error",
|
||||
// Swap the no-unused-expressions rule with a more chai-friendly one
|
||||
"no-unused-expressions": 0,
|
||||
"chai-friendly/no-unused-expressions": "error"
|
||||
}
|
||||
}
|
||||
4
services/spelling/.gitignore
vendored
4
services/spelling/.gitignore
vendored
|
|
@ -1,9 +1,5 @@
|
|||
**.swp
|
||||
**.swo
|
||||
app/js/*
|
||||
app.js
|
||||
test/UnitTests/js/*
|
||||
node_modules/*
|
||||
test/unit/js/
|
||||
cache/spell.cache
|
||||
**/*.map
|
||||
|
|
|
|||
8
services/spelling/.prettierrc
Normal file
8
services/spelling/.prettierrc
Normal file
|
|
@ -0,0 +1,8 @@
|
|||
# This file was auto-generated, do not edit it directly.
|
||||
# Instead run bin/update_build_scripts from
|
||||
# https://github.com/sharelatex/sharelatex-dev-environment
|
||||
# Version: 1.1.21
|
||||
{
|
||||
"semi": false,
|
||||
"singleQuote": true
|
||||
}
|
||||
|
|
@ -10,7 +10,6 @@ RUN npm install --quiet
|
|||
COPY . /app
|
||||
|
||||
|
||||
RUN npm run compile:all
|
||||
|
||||
FROM node:6.16.0
|
||||
|
||||
|
|
|
|||
8
services/spelling/Jenkinsfile
vendored
8
services/spelling/Jenkinsfile
vendored
|
|
@ -36,6 +36,14 @@ pipeline {
|
|||
}
|
||||
}
|
||||
|
||||
// should be enabled once Node version is updated to >=8
|
||||
// stage('Linting') {
|
||||
// steps {
|
||||
// sh 'DOCKER_COMPOSE_FLAGS="-f docker-compose.ci.yml" make format'
|
||||
// sh 'DOCKER_COMPOSE_FLAGS="-f docker-compose.ci.yml" make lint'
|
||||
// }
|
||||
// }
|
||||
|
||||
stage('Unit Tests') {
|
||||
steps {
|
||||
sh 'DOCKER_COMPOSE_FLAGS="-f docker-compose.ci.yml" make test_unit'
|
||||
|
|
|
|||
|
|
@ -16,12 +16,17 @@ DOCKER_COMPOSE := BUILD_NUMBER=$(BUILD_NUMBER) \
|
|||
clean:
|
||||
docker rmi ci/$(PROJECT_NAME):$(BRANCH_NAME)-$(BUILD_NUMBER)
|
||||
docker rmi gcr.io/overleaf-ops/$(PROJECT_NAME):$(BRANCH_NAME)-$(BUILD_NUMBER)
|
||||
rm -f app.js
|
||||
rm -rf app/js
|
||||
rm -rf test/unit/js
|
||||
rm -rf test/acceptance/js
|
||||
|
||||
test: test_unit test_acceptance
|
||||
format:
|
||||
$(DOCKER_COMPOSE) run --rm test_unit npm run format
|
||||
|
||||
format_fix:
|
||||
$(DOCKER_COMPOSE) run --rm test_unit npm run format:fix
|
||||
|
||||
lint:
|
||||
$(DOCKER_COMPOSE) run --rm test_unit npm run lint
|
||||
|
||||
test: format lint test_unit test_acceptance
|
||||
|
||||
test_unit:
|
||||
@[ ! -d test/unit ] && echo "spelling has no unit tests" || $(DOCKER_COMPOSE) run --rm test_unit
|
||||
|
|
|
|||
|
|
@ -1,47 +0,0 @@
|
|||
metrics = require("metrics-sharelatex")
|
||||
metrics.initialize("spelling")
|
||||
|
||||
|
||||
Settings = require 'settings-sharelatex'
|
||||
logger = require 'logger-sharelatex'
|
||||
logger.initialize("spelling")
|
||||
if Settings.sentry?.dsn?
|
||||
logger.initializeErrorReporting(Settings.sentry.dsn)
|
||||
metrics.memory.monitor(logger)
|
||||
|
||||
SpellingAPIController = require './app/js/SpellingAPIController'
|
||||
express = require('express')
|
||||
Path = require("path")
|
||||
server = express()
|
||||
metrics.injectMetricsRoute(server)
|
||||
bodyParser = require('body-parser')
|
||||
HealthCheckController = require("./app/js/HealthCheckController")
|
||||
|
||||
|
||||
|
||||
server.use bodyParser.json(limit: "2mb")
|
||||
server.use metrics.http.monitor(logger)
|
||||
|
||||
server.del "/user/:user_id", SpellingAPIController.deleteDic
|
||||
server.get "/user/:user_id", SpellingAPIController.getDic
|
||||
server.post "/user/:user_id/check", SpellingAPIController.check
|
||||
server.post "/user/:user_id/learn", SpellingAPIController.learn
|
||||
server.get "/status", (req, res)->
|
||||
res.send(status:'spelling api is up')
|
||||
|
||||
server.get "/health_check", HealthCheckController.healthCheck
|
||||
|
||||
profiler = require "v8-profiler"
|
||||
server.get "/profile", (req, res) ->
|
||||
time = parseInt(req.query.time || "1000")
|
||||
profiler.startProfiling("test")
|
||||
setTimeout () ->
|
||||
profile = profiler.stopProfiling("test")
|
||||
res.json(profile)
|
||||
, time
|
||||
|
||||
host = Settings.internal?.spelling?.host || "localhost"
|
||||
port = Settings.internal?.spelling?.port || 3005
|
||||
server.listen port, host, (error) ->
|
||||
throw error if error?
|
||||
logger.info "spelling starting up, listening on #{host}:#{port}"
|
||||
68
services/spelling/app.js
Normal file
68
services/spelling/app.js
Normal file
|
|
@ -0,0 +1,68 @@
|
|||
/*
|
||||
* decaffeinate suggestions:
|
||||
* DS102: Remove unnecessary code created because of implicit returns
|
||||
* DS103: Rewrite code to no longer use __guard__
|
||||
* DS207: Consider shorter variations of null checks
|
||||
* Full docs: https://github.com/decaffeinate/decaffeinate/blob/master/docs/suggestions.md
|
||||
*/
|
||||
const metrics = require('metrics-sharelatex')
|
||||
metrics.initialize('spelling')
|
||||
|
||||
const Settings = require('settings-sharelatex')
|
||||
const logger = require('logger-sharelatex')
|
||||
logger.initialize('spelling')
|
||||
if ((Settings.sentry != null ? Settings.sentry.dsn : undefined) != null) {
|
||||
logger.initializeErrorReporting(Settings.sentry.dsn)
|
||||
}
|
||||
metrics.memory.monitor(logger)
|
||||
|
||||
const SpellingAPIController = require('./app/js/SpellingAPIController')
|
||||
const express = require('express')
|
||||
const server = express()
|
||||
metrics.injectMetricsRoute(server)
|
||||
const bodyParser = require('body-parser')
|
||||
const HealthCheckController = require('./app/js/HealthCheckController')
|
||||
|
||||
server.use(bodyParser.json({ limit: '2mb' }))
|
||||
server.use(metrics.http.monitor(logger))
|
||||
|
||||
server.del('/user/:user_id', SpellingAPIController.deleteDic)
|
||||
server.get('/user/:user_id', SpellingAPIController.getDic)
|
||||
server.post('/user/:user_id/check', SpellingAPIController.check)
|
||||
server.post('/user/:user_id/learn', SpellingAPIController.learn)
|
||||
server.get('/status', (req, res) => res.send({ status: 'spelling api is up' }))
|
||||
|
||||
server.get('/health_check', HealthCheckController.healthCheck)
|
||||
|
||||
const profiler = require('v8-profiler')
|
||||
server.get('/profile', function(req, res) {
|
||||
const time = parseInt(req.query.time || '1000')
|
||||
profiler.startProfiling('test')
|
||||
return setTimeout(function() {
|
||||
const profile = profiler.stopProfiling('test')
|
||||
return res.json(profile)
|
||||
}, time)
|
||||
})
|
||||
|
||||
const host =
|
||||
__guard__(
|
||||
Settings.internal != null ? Settings.internal.spelling : undefined,
|
||||
x => x.host
|
||||
) || 'localhost'
|
||||
const port =
|
||||
__guard__(
|
||||
Settings.internal != null ? Settings.internal.spelling : undefined,
|
||||
x1 => x1.port
|
||||
) || 3005
|
||||
server.listen(port, host, function(error) {
|
||||
if (error != null) {
|
||||
throw error
|
||||
}
|
||||
return logger.info(`spelling starting up, listening on ${host}:${port}`)
|
||||
})
|
||||
|
||||
function __guard__(value, transform) {
|
||||
return typeof value !== 'undefined' && value !== null
|
||||
? transform(value)
|
||||
: undefined
|
||||
}
|
||||
|
|
@ -1,119 +0,0 @@
|
|||
async = require "async"
|
||||
ASpellWorkerPool = require "./ASpellWorkerPool"
|
||||
LRU = require "lru-cache"
|
||||
logger = require 'logger-sharelatex'
|
||||
fs = require 'fs'
|
||||
settings = require("settings-sharelatex")
|
||||
Path = require("path")
|
||||
|
||||
OneMinute = 60 * 1000
|
||||
opts = {max:10000, maxAge: OneMinute * 60 * 10}
|
||||
cache = LRU(opts)
|
||||
|
||||
cacheFsPath = Path.resolve(settings.cacheDir, "spell.cache")
|
||||
cacheFsPathTmp = cacheFsPath + ".tmp"
|
||||
|
||||
# load any existing cache
|
||||
try
|
||||
oldCache = fs.readFileSync cacheFsPath
|
||||
cache.load JSON.parse(oldCache)
|
||||
catch err
|
||||
logger.log err:err, cacheFsPath:cacheFsPath, "could not load the cache file"
|
||||
|
||||
# write the cache every 30 minutes
|
||||
setInterval () ->
|
||||
dump = JSON.stringify cache.dump()
|
||||
fs.writeFile cacheFsPathTmp, dump, (err) ->
|
||||
if err?
|
||||
logger.log {err}, "error writing cache file"
|
||||
fs.unlink cacheFsPathTmp
|
||||
else
|
||||
fs.rename cacheFsPathTmp, cacheFsPath
|
||||
logger.log {len: dump.length, cacheFsPath:cacheFsPath}, "wrote cache file"
|
||||
, 30 * OneMinute
|
||||
|
||||
class ASpellRunner
|
||||
checkWords: (language, words, callback = (error, result) ->) ->
|
||||
@runAspellOnWords language, words, (error, output) =>
|
||||
return callback(error) if error?
|
||||
#output = @removeAspellHeader(output)
|
||||
suggestions = @getSuggestions(language, output)
|
||||
results = []
|
||||
hits = 0
|
||||
addToCache = {}
|
||||
for word, i in words
|
||||
key = language + ':' + word
|
||||
cached = cache.get(key)
|
||||
if cached?
|
||||
hits++
|
||||
if cached == true
|
||||
# valid word, no need to do anything
|
||||
continue
|
||||
else
|
||||
results.push index: i, suggestions: cached
|
||||
else
|
||||
if suggestions[key]?
|
||||
addToCache[key] = suggestions[key]
|
||||
results.push index: i, suggestions: suggestions[key]
|
||||
else
|
||||
# a valid word, but uncached
|
||||
addToCache[key] = true
|
||||
|
||||
# update the cache after processing all words, to avoid cache
|
||||
# changing while we use it
|
||||
for k, v of addToCache
|
||||
cache.set(k, v)
|
||||
|
||||
logger.info hits: hits, total: words.length, hitrate: (hits/words.length).toFixed(2), "cache hit rate"
|
||||
callback null, results
|
||||
|
||||
getSuggestions: (language, output) ->
|
||||
lines = output.split("\n")
|
||||
suggestions = {}
|
||||
for line in lines
|
||||
if line[0] == "&" # Suggestions found
|
||||
parts = line.split(" ")
|
||||
if parts.length > 1
|
||||
word = parts[1]
|
||||
suggestionsString = line.slice(line.indexOf(":") + 2)
|
||||
suggestions[language + ":" + word] = suggestionsString.split(", ")
|
||||
else if line[0] == "#" # No suggestions
|
||||
parts = line.split(" ")
|
||||
if parts.length > 1
|
||||
word = parts[1]
|
||||
suggestions[language + ":" + word] = []
|
||||
return suggestions
|
||||
|
||||
#removeAspellHeader: (output) -> output.slice(1)
|
||||
|
||||
runAspellOnWords: (language, words, callback = (error, output) ->) ->
|
||||
# send words to aspell, get back string output for those words
|
||||
# find a free pipe for the language (or start one)
|
||||
# send the words down the pipe
|
||||
# send an END marker that will generate a "*" line in the output
|
||||
# when the output pipe receives the "*" return the data sofar and reset the pipe to be available
|
||||
#
|
||||
# @open(language)
|
||||
# @captureOutput(callback)
|
||||
# @setTerseMode()
|
||||
# start = new Date()
|
||||
|
||||
newWord = {}
|
||||
for word in words
|
||||
newWord[word] = true if !newWord[word] && !cache.has(language + ':' + word)
|
||||
words = Object.keys(newWord)
|
||||
|
||||
if words.length
|
||||
WorkerPool.check(language, words, ASpell.ASPELL_TIMEOUT, callback)
|
||||
else
|
||||
callback null, ""
|
||||
|
||||
module.exports = ASpell =
|
||||
# The description of how to call aspell from another program can be found here:
|
||||
# http://aspell.net/man-html/Through-A-Pipe.html
|
||||
checkWords: (language, words, callback = (error, result) ->) ->
|
||||
runner = new ASpellRunner()
|
||||
runner.checkWords language, words, callback
|
||||
ASPELL_TIMEOUT : 4000
|
||||
|
||||
WorkerPool = new ASpellWorkerPool()
|
||||
|
|
@ -1,117 +0,0 @@
|
|||
child_process = require("child_process")
|
||||
logger = require 'logger-sharelatex'
|
||||
metrics = require('metrics-sharelatex')
|
||||
_ = require "underscore"
|
||||
|
||||
BATCH_SIZE = 100
|
||||
|
||||
class ASpellWorker
|
||||
constructor: (language) ->
|
||||
@language = language
|
||||
@count = 0
|
||||
@pipe = child_process.spawn("aspell", ["pipe", "-t", "--encoding=utf-8", "-d", language])
|
||||
logger.info process: @pipe.pid, lang: @language, "starting new aspell worker"
|
||||
metrics.inc "aspellWorker", 1, {status: "start", method: @language}
|
||||
@pipe.on 'exit', () =>
|
||||
@state = 'killed'
|
||||
logger.info process: @pipe.pid, lang: @language, "aspell worker has exited"
|
||||
metrics.inc "aspellWorker" , 1, {status: "exit", method: @language}
|
||||
@pipe.on 'close', () =>
|
||||
@state = 'closed' unless @state == 'killed'
|
||||
if @callback?
|
||||
logger.err process: @pipe.pid, lang: @language, "aspell worker closed output streams with uncalled callback"
|
||||
@callback new Error("aspell worker closed output streams with uncalled callback"), []
|
||||
@callback = null
|
||||
@pipe.on 'error', (err) =>
|
||||
@state = 'error' unless @state == 'killed'
|
||||
logger.log process: @pipe.pid, error: err, stdout: output.slice(-1024), stderr: error.slice(-1024), lang: @language, "aspell worker error"
|
||||
if @callback?
|
||||
@callback err, []
|
||||
@callback = null
|
||||
@pipe.stdin.on 'error', (err) =>
|
||||
@state = 'error' unless @state == 'killed'
|
||||
logger.info process: @pipe.pid, error: err, stdout: output.slice(-1024), stderr: error.slice(-1024), lang: @language, "aspell worker error on stdin"
|
||||
if @callback?
|
||||
@callback err, []
|
||||
@callback = null
|
||||
|
||||
output = ""
|
||||
endMarker = new RegExp("^[a-z][a-z]", "m")
|
||||
@pipe.stdout.on "data", (chunk) =>
|
||||
output = output + chunk
|
||||
# We receive the language code from Aspell as the end of data marker
|
||||
if chunk.toString().match(endMarker)
|
||||
if @callback?
|
||||
@callback(null, output.slice())
|
||||
@callback = null # only allow one callback in use
|
||||
else
|
||||
logger.err process: @pipe.pid, lang: @language, "end of data marker received when callback already used"
|
||||
@state = 'ready'
|
||||
output = ""
|
||||
error = ""
|
||||
|
||||
error = ""
|
||||
@pipe.stderr.on "data", (chunk) =>
|
||||
error = error + chunk
|
||||
|
||||
@pipe.stdout.on "end", () =>
|
||||
# process has ended
|
||||
@state = "end"
|
||||
|
||||
isReady: () ->
|
||||
return @state == 'ready'
|
||||
|
||||
check: (words, callback) ->
|
||||
# we will now send data to aspell, and be ready again when we
|
||||
# receive the end of data marker
|
||||
@state = 'busy'
|
||||
if @callback? # only allow one callback in use
|
||||
logger.err process: @pipe.pid, lang: @language, "CALLBACK ALREADY IN USE"
|
||||
return @callback new Error("Aspell callback already in use - SHOULD NOT HAPPEN")
|
||||
@callback = _.once callback # extra defence against double callback
|
||||
@setTerseMode()
|
||||
@write(words)
|
||||
@flush()
|
||||
|
||||
write: (words) ->
|
||||
i = 0
|
||||
while i < words.length
|
||||
# batch up the words to check for efficiency
|
||||
batch = words.slice(i, i + BATCH_SIZE)
|
||||
@sendWords batch
|
||||
i += BATCH_SIZE
|
||||
|
||||
flush: () ->
|
||||
# get aspell to send an end of data marker "*" when ready
|
||||
#@sendCommand("%") # take the aspell pipe out of terse mode so we can look for a '*'
|
||||
#@sendCommand("^ENDOFSTREAMMARKER") # send our marker which will generate a '*'
|
||||
#@sendCommand("!") # go back into terse mode
|
||||
@sendCommand("$$l")
|
||||
|
||||
shutdown: (reason) ->
|
||||
logger.info process: @pipe.pid, reason: reason, 'shutting down'
|
||||
@state = "closing"
|
||||
@pipe.stdin.end()
|
||||
|
||||
kill: (reason) ->
|
||||
logger.info process: @pipe.pid, reason: reason, 'killing'
|
||||
return if @state == 'killed'
|
||||
@pipe.kill('SIGKILL')
|
||||
|
||||
setTerseMode: () ->
|
||||
@sendCommand("!")
|
||||
|
||||
sendWord: (word) ->
|
||||
@sendCommand("^" + word)
|
||||
|
||||
sendWords: (words) ->
|
||||
# Aspell accepts multiple words to check on the same line
|
||||
# ^word1 word2 word3 ...
|
||||
# See aspell.info, writing programs to use Aspell Through A Pipe
|
||||
@sendCommand("^" + words.join(" "))
|
||||
@count++
|
||||
|
||||
sendCommand: (command) ->
|
||||
@pipe.stdin.write(command + "\n")
|
||||
|
||||
module.exports = ASpellWorker
|
||||
|
|
@ -1,76 +0,0 @@
|
|||
ASpellWorker = require "./ASpellWorker"
|
||||
_ = require "underscore"
|
||||
logger = require 'logger-sharelatex'
|
||||
metrics = require('metrics-sharelatex')
|
||||
|
||||
class ASpellWorkerPool
|
||||
MAX_REQUESTS: 100*1024
|
||||
MAX_WORKERS: 32
|
||||
MAX_IDLE_TIME: 1000
|
||||
MAX_REQUEST_TIME: 60*1000
|
||||
|
||||
constructor: (@options) ->
|
||||
@PROCESS_POOL = []
|
||||
|
||||
create: (language) ->
|
||||
if @PROCESS_POOL.length >= @MAX_WORKERS
|
||||
logger.log maxworkers: @MAX_WORKERS, "maximum number of workers already running"
|
||||
return null
|
||||
worker = new ASpellWorker(language, @options)
|
||||
worker.pipe.on 'exit', () =>
|
||||
if worker.killTimer?
|
||||
clearTimeout worker.killTimer
|
||||
worker.killTimer = null
|
||||
if worker.idleTimer?
|
||||
clearTimeout worker.idleTimer
|
||||
worker.idleTimer = null
|
||||
logger.info process: worker.pipe.pid, lang: language, "removing aspell worker from pool"
|
||||
@cleanup()
|
||||
@PROCESS_POOL.push(worker)
|
||||
metrics.gauge 'aspellWorkerPool-size', @PROCESS_POOL.length
|
||||
return worker
|
||||
|
||||
cleanup: () ->
|
||||
active = @PROCESS_POOL.filter (worker) ->
|
||||
worker.state != 'killed'
|
||||
@PROCESS_POOL = active
|
||||
metrics.gauge 'aspellWorkerPool-size', @PROCESS_POOL.length
|
||||
|
||||
check: (language, words, timeout, callback) ->
|
||||
# look for an existing process in the pool
|
||||
availableWorker = _.find @PROCESS_POOL, (cached) ->
|
||||
cached.language == language && cached.isReady()
|
||||
if not availableWorker?
|
||||
worker = @create(language)
|
||||
else
|
||||
worker = availableWorker
|
||||
|
||||
if not worker?
|
||||
# return error if too many workers
|
||||
callback(new Error("no worker available"))
|
||||
return
|
||||
|
||||
if worker.idleTimer?
|
||||
clearTimeout worker.idleTimer
|
||||
worker.idleTimer = null
|
||||
|
||||
worker.killTimer = setTimeout () ->
|
||||
worker.kill("spell check timed out")
|
||||
, timeout || @MAX_REQUEST_TIME
|
||||
|
||||
worker.check words, (err, output) =>
|
||||
if worker.killTimer?
|
||||
clearTimeout worker.killTimer
|
||||
worker.killTimer = null
|
||||
callback(err, output)
|
||||
return if err? # process has shut down
|
||||
if worker.count > @MAX_REQUESTS
|
||||
worker.shutdown("reached limit of " + @MAX_REQUESTS + " requests")
|
||||
else
|
||||
# queue a shutdown if worker is idle
|
||||
worker.idleTimer = setTimeout () ->
|
||||
worker.shutdown("idle worker")
|
||||
worker.idleTimer = null
|
||||
, @MAX_IDLE_TIME
|
||||
|
||||
module.exports = ASpellWorkerPool
|
||||
|
|
@ -1,4 +0,0 @@
|
|||
MongoJS = require "mongojs"
|
||||
Settings = require "settings-sharelatex"
|
||||
module.exports = MongoJS(Settings.mongo.url, ["spellingPreferences"])
|
||||
|
||||
|
|
@ -1,23 +0,0 @@
|
|||
request = require("request")
|
||||
logger = require 'logger-sharelatex'
|
||||
settings = require 'settings-sharelatex'
|
||||
|
||||
module.exports =
|
||||
|
||||
healthCheck: (req, res)->
|
||||
opts =
|
||||
url: "http://localhost:3005/user/#{settings.healthCheckUserId}/check"
|
||||
json:
|
||||
words:["helllo"]
|
||||
language: "en"
|
||||
timeout: 1000 * 20
|
||||
request.post opts, (err, response, body)->
|
||||
if err?
|
||||
return res.sendStatus 500
|
||||
numberOfSuggestions = body?.misspellings?[0]?.suggestions?.length
|
||||
if numberOfSuggestions > 10
|
||||
logger.log "health check passed"
|
||||
res.sendStatus 200
|
||||
else
|
||||
logger.err body:body, numberOfSuggestions:numberOfSuggestions, "health check failed"
|
||||
res.sendStatus 500
|
||||
|
|
@ -1,40 +0,0 @@
|
|||
db = require("./DB")
|
||||
mongoCache = require("./MongoCache")
|
||||
logger = require 'logger-sharelatex'
|
||||
metrics = require('metrics-sharelatex')
|
||||
|
||||
module.exports = LearnedWordsManager =
|
||||
learnWord: (user_token, word, callback = (error)->) ->
|
||||
mongoCache.del(user_token)
|
||||
db.spellingPreferences.update {
|
||||
token: user_token
|
||||
}, {
|
||||
$push: learnedWords: word
|
||||
}, {
|
||||
upsert: true
|
||||
}, callback
|
||||
|
||||
getLearnedWords: (user_token, callback = (error, words)->) ->
|
||||
mongoCachedWords = mongoCache.get(user_token)
|
||||
if mongoCachedWords?
|
||||
metrics.inc "mongoCache", 0.1, {status: "hit"}
|
||||
return callback(null, mongoCachedWords)
|
||||
|
||||
metrics.inc "mongoCache", 0.1, {status: "miss"}
|
||||
logger.info user_token:user_token, "mongoCache miss"
|
||||
|
||||
db.spellingPreferences.findOne token: user_token, (error, preferences) ->
|
||||
return callback error if error?
|
||||
words = preferences?.learnedWords || []
|
||||
mongoCache.set(user_token, words)
|
||||
callback null, words
|
||||
|
||||
deleteUsersLearnedWords: (user_token, callback =(error)->)->
|
||||
db.spellingPreferences.remove token: user_token, callback
|
||||
|
||||
|
||||
[
|
||||
'learnWord',
|
||||
'getLearnedWords'
|
||||
].map (method) ->
|
||||
metrics.timeAsyncMethod(LearnedWordsManager, method, 'mongo.LearnedWordsManager', logger)
|
||||
|
|
@ -1,8 +0,0 @@
|
|||
LRU = require("lru-cache")
|
||||
cacheOpts =
|
||||
max: 15000
|
||||
maxAge: 1000 * 60 * 60 * 10
|
||||
|
||||
cache = LRU(cacheOpts)
|
||||
|
||||
module.exports = cache
|
||||
|
|
@ -1,35 +0,0 @@
|
|||
SpellingAPIManager = require './SpellingAPIManager'
|
||||
logger = require 'logger-sharelatex'
|
||||
metrics = require('metrics-sharelatex')
|
||||
|
||||
module.exports = SpellingAPIController =
|
||||
check: (req, res, next) ->
|
||||
metrics.inc "spelling-check", 0.1
|
||||
logger.info token: req?.params?.user_id, word_count: req?.body?.words?.length, "running check"
|
||||
SpellingAPIManager.runRequest req.params.user_id, req.body, (error, result) ->
|
||||
if error?
|
||||
logger.err err:error, user_id:req?.params?.user_id, word_count: req?.body?.words?.length, "error processing spelling request"
|
||||
return res.sendStatus(500)
|
||||
res.send(result)
|
||||
|
||||
learn: (req, res, next) ->
|
||||
metrics.inc "spelling-learn", 0.1
|
||||
logger.info token: req?.params?.user_id, word: req?.body?.word, "learning word"
|
||||
SpellingAPIManager.learnWord req.params.user_id, req.body, (error, result) ->
|
||||
return next(error) if error?
|
||||
res.sendStatus(200)
|
||||
next()
|
||||
|
||||
|
||||
deleteDic: (req, res, next)->
|
||||
logger.log token: req?.params?.user_id, word: req?.body?.word, "deleting user dictionary"
|
||||
SpellingAPIManager.deleteDic req.params.user_id, (error) ->
|
||||
return next(error) if error?
|
||||
res.sendStatus(204)
|
||||
|
||||
|
||||
getDic: (req, res, next)->
|
||||
logger.info token: req?.params?.user_id, "getting user dictionary"
|
||||
SpellingAPIManager.getDic req.params.user_id, (error, words)->
|
||||
return next(error) if error?
|
||||
res.send(words)
|
||||
|
|
@ -1,54 +0,0 @@
|
|||
ASpell = require './ASpell'
|
||||
LearnedWordsManager = require './LearnedWordsManager'
|
||||
async = require 'async'
|
||||
|
||||
module.exports = SpellingAPIManager =
|
||||
|
||||
whitelist: [
|
||||
'ShareLaTeX',
|
||||
'sharelatex',
|
||||
'LaTeX',
|
||||
'http',
|
||||
'https',
|
||||
'www'
|
||||
]
|
||||
|
||||
runRequest: (token, request, callback = (error, result) ->) ->
|
||||
if !request.words?
|
||||
return callback(new Error("malformed JSON"))
|
||||
|
||||
lang = request.language || "en"
|
||||
|
||||
check = (words, callback) ->
|
||||
ASpell.checkWords lang, words, (error, misspellings) ->
|
||||
callback error, misspellings: misspellings
|
||||
|
||||
wordsToCheck = request.words || []
|
||||
|
||||
if token?
|
||||
LearnedWordsManager.getLearnedWords token, (error, learnedWords) ->
|
||||
return callback(error) if error?
|
||||
words = (wordsToCheck).slice(0,10000)
|
||||
check words, (error, result) ->
|
||||
return callback error if error?
|
||||
result.misspellings = result.misspellings.filter (m) ->
|
||||
word = words[m.index]
|
||||
learnedWords.indexOf(word) == -1 and SpellingAPIManager.whitelist.indexOf(word) == -1
|
||||
callback error, result
|
||||
else
|
||||
check(wordsToCheck, callback)
|
||||
|
||||
learnWord: (token, request, callback = (error) ->) ->
|
||||
if !request.word?
|
||||
return callback(new Error("malformed JSON"))
|
||||
if !token?
|
||||
return callback(new Error("no token provided"))
|
||||
|
||||
LearnedWordsManager.learnWord token, request.word, callback
|
||||
|
||||
deleteDic: (token, callback)->
|
||||
LearnedWordsManager.deleteUsersLearnedWords token, callback
|
||||
|
||||
getDic: (token, callback)->
|
||||
LearnedWordsManager.getLearnedWords token, callback
|
||||
|
||||
175
services/spelling/app/js/ASpell.js
Normal file
175
services/spelling/app/js/ASpell.js
Normal file
|
|
@ -0,0 +1,175 @@
|
|||
// TODO: This file was created by bulk-decaffeinate.
|
||||
// Sanity-check the conversion and remove this comment.
|
||||
/*
|
||||
* decaffeinate suggestions:
|
||||
* DS101: Remove unnecessary use of Array.from
|
||||
* DS102: Remove unnecessary code created because of implicit returns
|
||||
* DS207: Consider shorter variations of null checks
|
||||
* Full docs: https://github.com/decaffeinate/decaffeinate/blob/master/docs/suggestions.md
|
||||
*/
|
||||
let ASpell
|
||||
const ASpellWorkerPool = require('./ASpellWorkerPool')
|
||||
const LRU = require('lru-cache')
|
||||
const logger = require('logger-sharelatex')
|
||||
const fs = require('fs')
|
||||
const settings = require('settings-sharelatex')
|
||||
const Path = require('path')
|
||||
|
||||
const OneMinute = 60 * 1000
|
||||
const opts = { max: 10000, maxAge: OneMinute * 60 * 10 }
|
||||
const cache = LRU(opts)
|
||||
|
||||
const cacheFsPath = Path.resolve(settings.cacheDir, 'spell.cache')
|
||||
const cacheFsPathTmp = cacheFsPath + '.tmp'
|
||||
|
||||
// load any existing cache
|
||||
try {
|
||||
const oldCache = fs.readFileSync(cacheFsPath)
|
||||
cache.load(JSON.parse(oldCache))
|
||||
} catch (error) {
|
||||
const err = error
|
||||
logger.log({ err, cacheFsPath }, 'could not load the cache file')
|
||||
}
|
||||
|
||||
// write the cache every 30 minutes
|
||||
setInterval(function() {
|
||||
const dump = JSON.stringify(cache.dump())
|
||||
return fs.writeFile(cacheFsPathTmp, dump, function(err) {
|
||||
if (err != null) {
|
||||
logger.log({ err }, 'error writing cache file')
|
||||
return fs.unlink(cacheFsPathTmp)
|
||||
} else {
|
||||
fs.rename(cacheFsPathTmp, cacheFsPath)
|
||||
return logger.log({ len: dump.length, cacheFsPath }, 'wrote cache file')
|
||||
}
|
||||
})
|
||||
}, 30 * OneMinute)
|
||||
|
||||
class ASpellRunner {
|
||||
checkWords(language, words, callback) {
|
||||
if (callback == null) {
|
||||
callback = () => {}
|
||||
}
|
||||
return this.runAspellOnWords(language, words, (error, output) => {
|
||||
if (error != null) {
|
||||
return callback(error)
|
||||
}
|
||||
// output = @removeAspellHeader(output)
|
||||
const suggestions = this.getSuggestions(language, output)
|
||||
const results = []
|
||||
let hits = 0
|
||||
const addToCache = {}
|
||||
for (let i = 0; i < words.length; i++) {
|
||||
const word = words[i]
|
||||
const key = language + ':' + word
|
||||
const cached = cache.get(key)
|
||||
if (cached != null) {
|
||||
hits++
|
||||
if (cached === true) {
|
||||
// valid word, no need to do anything
|
||||
continue
|
||||
} else {
|
||||
results.push({ index: i, suggestions: cached })
|
||||
}
|
||||
} else {
|
||||
if (suggestions[key] != null) {
|
||||
addToCache[key] = suggestions[key]
|
||||
results.push({ index: i, suggestions: suggestions[key] })
|
||||
} else {
|
||||
// a valid word, but uncached
|
||||
addToCache[key] = true
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// update the cache after processing all words, to avoid cache
|
||||
// changing while we use it
|
||||
for (let k in addToCache) {
|
||||
const v = addToCache[k]
|
||||
cache.set(k, v)
|
||||
}
|
||||
|
||||
logger.info(
|
||||
{
|
||||
hits,
|
||||
total: words.length,
|
||||
hitrate: (hits / words.length).toFixed(2)
|
||||
},
|
||||
'cache hit rate'
|
||||
)
|
||||
return callback(null, results)
|
||||
})
|
||||
}
|
||||
|
||||
getSuggestions(language, output) {
|
||||
const lines = output.split('\n')
|
||||
const suggestions = {}
|
||||
for (let line of Array.from(lines)) {
|
||||
var parts, word
|
||||
if (line[0] === '&') {
|
||||
// Suggestions found
|
||||
parts = line.split(' ')
|
||||
if (parts.length > 1) {
|
||||
word = parts[1]
|
||||
const suggestionsString = line.slice(line.indexOf(':') + 2)
|
||||
suggestions[language + ':' + word] = suggestionsString.split(', ')
|
||||
}
|
||||
} else if (line[0] === '#') {
|
||||
// No suggestions
|
||||
parts = line.split(' ')
|
||||
if (parts.length > 1) {
|
||||
word = parts[1]
|
||||
suggestions[language + ':' + word] = []
|
||||
}
|
||||
}
|
||||
}
|
||||
return suggestions
|
||||
}
|
||||
|
||||
// removeAspellHeader: (output) -> output.slice(1)
|
||||
|
||||
runAspellOnWords(language, words, callback) {
|
||||
// send words to aspell, get back string output for those words
|
||||
// find a free pipe for the language (or start one)
|
||||
// send the words down the pipe
|
||||
// send an END marker that will generate a "*" line in the output
|
||||
// when the output pipe receives the "*" return the data sofar and reset the pipe to be available
|
||||
//
|
||||
// @open(language)
|
||||
// @captureOutput(callback)
|
||||
// @setTerseMode()
|
||||
// start = new Date()
|
||||
|
||||
if (callback == null) {
|
||||
callback = () => {}
|
||||
}
|
||||
const newWord = {}
|
||||
for (let word of Array.from(words)) {
|
||||
if (!newWord[word] && !cache.has(language + ':' + word)) {
|
||||
newWord[word] = true
|
||||
}
|
||||
}
|
||||
words = Object.keys(newWord)
|
||||
|
||||
if (words.length) {
|
||||
return WorkerPool.check(language, words, ASpell.ASPELL_TIMEOUT, callback)
|
||||
} else {
|
||||
return callback(null, '')
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
module.exports = ASpell = {
|
||||
// The description of how to call aspell from another program can be found here:
|
||||
// http://aspell.net/man-html/Through-A-Pipe.html
|
||||
checkWords(language, words, callback) {
|
||||
if (callback == null) {
|
||||
callback = () => {}
|
||||
}
|
||||
const runner = new ASpellRunner()
|
||||
return runner.checkWords(language, words, callback)
|
||||
},
|
||||
ASPELL_TIMEOUT: 4000
|
||||
}
|
||||
|
||||
var WorkerPool = new ASpellWorkerPool()
|
||||
213
services/spelling/app/js/ASpellWorker.js
Normal file
213
services/spelling/app/js/ASpellWorker.js
Normal file
|
|
@ -0,0 +1,213 @@
|
|||
// TODO: This file was created by bulk-decaffeinate.
|
||||
// Sanity-check the conversion and remove this comment.
|
||||
/*
|
||||
* decaffeinate suggestions:
|
||||
* DS102: Remove unnecessary code created because of implicit returns
|
||||
* DS205: Consider reworking code to avoid use of IIFEs
|
||||
* DS207: Consider shorter variations of null checks
|
||||
* Full docs: https://github.com/decaffeinate/decaffeinate/blob/master/docs/suggestions.md
|
||||
*/
|
||||
const childProcess = require('child_process')
|
||||
const logger = require('logger-sharelatex')
|
||||
const metrics = require('metrics-sharelatex')
|
||||
const _ = require('underscore')
|
||||
|
||||
const BATCH_SIZE = 100
|
||||
|
||||
class ASpellWorker {
|
||||
constructor(language) {
|
||||
this.language = language
|
||||
this.count = 0
|
||||
this.pipe = childProcess.spawn('aspell', [
|
||||
'pipe',
|
||||
'-t',
|
||||
'--encoding=utf-8',
|
||||
'-d',
|
||||
language
|
||||
])
|
||||
logger.info(
|
||||
{ process: this.pipe.pid, lang: this.language },
|
||||
'starting new aspell worker'
|
||||
)
|
||||
metrics.inc('aspellWorker', 1, { status: 'start', method: this.language })
|
||||
this.pipe.on('exit', () => {
|
||||
this.state = 'killed'
|
||||
logger.info(
|
||||
{ process: this.pipe.pid, lang: this.language },
|
||||
'aspell worker has exited'
|
||||
)
|
||||
return metrics.inc('aspellWorker', 1, {
|
||||
status: 'exit',
|
||||
method: this.language
|
||||
})
|
||||
})
|
||||
this.pipe.on('close', () => {
|
||||
if (this.state !== 'killed') {
|
||||
this.state = 'closed'
|
||||
}
|
||||
if (this.callback != null) {
|
||||
logger.err(
|
||||
{ process: this.pipe.pid, lang: this.language },
|
||||
'aspell worker closed output streams with uncalled callback'
|
||||
)
|
||||
this.callback(
|
||||
new Error(
|
||||
'aspell worker closed output streams with uncalled callback'
|
||||
),
|
||||
[]
|
||||
)
|
||||
return (this.callback = null)
|
||||
}
|
||||
})
|
||||
this.pipe.on('error', err => {
|
||||
if (this.state !== 'killed') {
|
||||
this.state = 'error'
|
||||
}
|
||||
logger.log(
|
||||
{
|
||||
process: this.pipe.pid,
|
||||
error: err,
|
||||
stdout: output.slice(-1024),
|
||||
stderr: error.slice(-1024),
|
||||
lang: this.language
|
||||
},
|
||||
'aspell worker error'
|
||||
)
|
||||
if (this.callback != null) {
|
||||
this.callback(err, [])
|
||||
return (this.callback = null)
|
||||
}
|
||||
})
|
||||
this.pipe.stdin.on('error', err => {
|
||||
if (this.state !== 'killed') {
|
||||
this.state = 'error'
|
||||
}
|
||||
logger.info(
|
||||
{
|
||||
process: this.pipe.pid,
|
||||
error: err,
|
||||
stdout: output.slice(-1024),
|
||||
stderr: error.slice(-1024),
|
||||
lang: this.language
|
||||
},
|
||||
'aspell worker error on stdin'
|
||||
)
|
||||
if (this.callback != null) {
|
||||
this.callback(err, [])
|
||||
return (this.callback = null)
|
||||
}
|
||||
})
|
||||
|
||||
var output = ''
|
||||
const endMarker = new RegExp('^[a-z][a-z]', 'm')
|
||||
this.pipe.stdout.on('data', chunk => {
|
||||
output = output + chunk
|
||||
// We receive the language code from Aspell as the end of data marker
|
||||
if (chunk.toString().match(endMarker)) {
|
||||
if (this.callback != null) {
|
||||
this.callback(null, output.slice())
|
||||
this.callback = null // only allow one callback in use
|
||||
} else {
|
||||
logger.err(
|
||||
{ process: this.pipe.pid, lang: this.language },
|
||||
'end of data marker received when callback already used'
|
||||
)
|
||||
}
|
||||
this.state = 'ready'
|
||||
output = ''
|
||||
}
|
||||
})
|
||||
|
||||
var error = ''
|
||||
this.pipe.stderr.on('data', chunk => {
|
||||
return (error = error + chunk)
|
||||
})
|
||||
|
||||
this.pipe.stdout.on('end', () => {
|
||||
// process has ended
|
||||
return (this.state = 'end')
|
||||
})
|
||||
}
|
||||
|
||||
isReady() {
|
||||
return this.state === 'ready'
|
||||
}
|
||||
|
||||
check(words, callback) {
|
||||
// we will now send data to aspell, and be ready again when we
|
||||
// receive the end of data marker
|
||||
this.state = 'busy'
|
||||
if (this.callback != null) {
|
||||
// only allow one callback in use
|
||||
logger.err(
|
||||
{ process: this.pipe.pid, lang: this.language },
|
||||
'CALLBACK ALREADY IN USE'
|
||||
)
|
||||
return this.callback(
|
||||
new Error('Aspell callback already in use - SHOULD NOT HAPPEN')
|
||||
)
|
||||
}
|
||||
this.callback = _.once(callback) // extra defence against double callback
|
||||
this.setTerseMode()
|
||||
this.write(words)
|
||||
return this.flush()
|
||||
}
|
||||
|
||||
write(words) {
|
||||
let i = 0
|
||||
return (() => {
|
||||
const result = []
|
||||
while (i < words.length) {
|
||||
// batch up the words to check for efficiency
|
||||
const batch = words.slice(i, i + BATCH_SIZE)
|
||||
this.sendWords(batch)
|
||||
result.push((i += BATCH_SIZE))
|
||||
}
|
||||
return result
|
||||
})()
|
||||
}
|
||||
|
||||
flush() {
|
||||
// get aspell to send an end of data marker "*" when ready
|
||||
// @sendCommand("%") # take the aspell pipe out of terse mode so we can look for a '*'
|
||||
// @sendCommand("^ENDOFSTREAMMARKER") # send our marker which will generate a '*'
|
||||
// @sendCommand("!") # go back into terse mode
|
||||
return this.sendCommand('$$l')
|
||||
}
|
||||
|
||||
shutdown(reason) {
|
||||
logger.info({ process: this.pipe.pid, reason }, 'shutting down')
|
||||
this.state = 'closing'
|
||||
return this.pipe.stdin.end()
|
||||
}
|
||||
|
||||
kill(reason) {
|
||||
logger.info({ process: this.pipe.pid, reason }, 'killing')
|
||||
if (this.state === 'killed') {
|
||||
return
|
||||
}
|
||||
return this.pipe.kill('SIGKILL')
|
||||
}
|
||||
|
||||
setTerseMode() {
|
||||
return this.sendCommand('!')
|
||||
}
|
||||
|
||||
sendWord(word) {
|
||||
return this.sendCommand(`^${word}`)
|
||||
}
|
||||
|
||||
sendWords(words) {
|
||||
// Aspell accepts multiple words to check on the same line
|
||||
// ^word1 word2 word3 ...
|
||||
// See aspell.info, writing programs to use Aspell Through A Pipe
|
||||
this.sendCommand(`^${words.join(' ')}`)
|
||||
return this.count++
|
||||
}
|
||||
|
||||
sendCommand(command) {
|
||||
return this.pipe.stdin.write(command + '\n')
|
||||
}
|
||||
}
|
||||
|
||||
module.exports = ASpellWorker
|
||||
115
services/spelling/app/js/ASpellWorkerPool.js
Normal file
115
services/spelling/app/js/ASpellWorkerPool.js
Normal file
|
|
@ -0,0 +1,115 @@
|
|||
// TODO: This file was created by bulk-decaffeinate.
|
||||
// Sanity-check the conversion and remove this comment.
|
||||
/*
|
||||
* decaffeinate suggestions:
|
||||
* DS102: Remove unnecessary code created because of implicit returns
|
||||
* DS206: Consider reworking classes to avoid initClass
|
||||
* DS207: Consider shorter variations of null checks
|
||||
* Full docs: https://github.com/decaffeinate/decaffeinate/blob/master/docs/suggestions.md
|
||||
*/
|
||||
const ASpellWorker = require('./ASpellWorker')
|
||||
const _ = require('underscore')
|
||||
const logger = require('logger-sharelatex')
|
||||
const metrics = require('metrics-sharelatex')
|
||||
|
||||
class ASpellWorkerPool {
|
||||
static initClass() {
|
||||
this.prototype.MAX_REQUESTS = 100 * 1024
|
||||
this.prototype.MAX_WORKERS = 32
|
||||
this.prototype.MAX_IDLE_TIME = 1000
|
||||
this.prototype.MAX_REQUEST_TIME = 60 * 1000
|
||||
}
|
||||
|
||||
constructor(options) {
|
||||
this.options = options
|
||||
this.PROCESS_POOL = []
|
||||
}
|
||||
|
||||
create(language) {
|
||||
if (this.PROCESS_POOL.length >= this.MAX_WORKERS) {
|
||||
logger.log(
|
||||
{ maxworkers: this.MAX_WORKERS },
|
||||
'maximum number of workers already running'
|
||||
)
|
||||
return null
|
||||
}
|
||||
const worker = new ASpellWorker(language, this.options)
|
||||
worker.pipe.on('exit', () => {
|
||||
if (worker.killTimer != null) {
|
||||
clearTimeout(worker.killTimer)
|
||||
worker.killTimer = null
|
||||
}
|
||||
if (worker.idleTimer != null) {
|
||||
clearTimeout(worker.idleTimer)
|
||||
worker.idleTimer = null
|
||||
}
|
||||
logger.info(
|
||||
{ process: worker.pipe.pid, lang: language },
|
||||
'removing aspell worker from pool'
|
||||
)
|
||||
return this.cleanup()
|
||||
})
|
||||
this.PROCESS_POOL.push(worker)
|
||||
metrics.gauge('aspellWorkerPool-size', this.PROCESS_POOL.length)
|
||||
return worker
|
||||
}
|
||||
|
||||
cleanup() {
|
||||
const active = this.PROCESS_POOL.filter(worker => worker.state !== 'killed')
|
||||
this.PROCESS_POOL = active
|
||||
return metrics.gauge('aspellWorkerPool-size', this.PROCESS_POOL.length)
|
||||
}
|
||||
|
||||
check(language, words, timeout, callback) {
|
||||
// look for an existing process in the pool
|
||||
let worker
|
||||
const availableWorker = _.find(
|
||||
this.PROCESS_POOL,
|
||||
cached => cached.language === language && cached.isReady()
|
||||
)
|
||||
if (availableWorker == null) {
|
||||
worker = this.create(language)
|
||||
} else {
|
||||
worker = availableWorker
|
||||
}
|
||||
|
||||
if (worker == null) {
|
||||
// return error if too many workers
|
||||
callback(new Error('no worker available'))
|
||||
return
|
||||
}
|
||||
|
||||
if (worker.idleTimer != null) {
|
||||
clearTimeout(worker.idleTimer)
|
||||
worker.idleTimer = null
|
||||
}
|
||||
|
||||
worker.killTimer = setTimeout(
|
||||
() => worker.kill('spell check timed out'),
|
||||
timeout || this.MAX_REQUEST_TIME
|
||||
)
|
||||
|
||||
return worker.check(words, (err, output) => {
|
||||
if (worker.killTimer != null) {
|
||||
clearTimeout(worker.killTimer)
|
||||
worker.killTimer = null
|
||||
}
|
||||
callback(err, output)
|
||||
if (err != null) {
|
||||
return
|
||||
} // process has shut down
|
||||
if (worker.count > this.MAX_REQUESTS) {
|
||||
return worker.shutdown(`reached limit of ${this.MAX_REQUESTS} requests`)
|
||||
} else {
|
||||
// queue a shutdown if worker is idle
|
||||
return (worker.idleTimer = setTimeout(function() {
|
||||
worker.shutdown('idle worker')
|
||||
return (worker.idleTimer = null)
|
||||
}, this.MAX_IDLE_TIME))
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
ASpellWorkerPool.initClass()
|
||||
|
||||
module.exports = ASpellWorkerPool
|
||||
5
services/spelling/app/js/DB.js
Normal file
5
services/spelling/app/js/DB.js
Normal file
|
|
@ -0,0 +1,5 @@
|
|||
// TODO: This file was created by bulk-decaffeinate.
|
||||
// Sanity-check the conversion and remove this comment.
|
||||
const MongoJS = require('mongojs')
|
||||
const Settings = require('settings-sharelatex')
|
||||
module.exports = MongoJS(Settings.mongo.url, ['spellingPreferences'])
|
||||
50
services/spelling/app/js/HealthCheckController.js
Normal file
50
services/spelling/app/js/HealthCheckController.js
Normal file
|
|
@ -0,0 +1,50 @@
|
|||
// TODO: This file was created by bulk-decaffeinate.
|
||||
// Sanity-check the conversion and remove this comment.
|
||||
/*
|
||||
* decaffeinate suggestions:
|
||||
* DS102: Remove unnecessary code created because of implicit returns
|
||||
* DS103: Rewrite code to no longer use __guard__
|
||||
* DS207: Consider shorter variations of null checks
|
||||
* Full docs: https://github.com/decaffeinate/decaffeinate/blob/master/docs/suggestions.md
|
||||
*/
|
||||
const request = require('request')
|
||||
const logger = require('logger-sharelatex')
|
||||
const settings = require('settings-sharelatex')
|
||||
|
||||
module.exports = {
|
||||
healthCheck(req, res) {
|
||||
const opts = {
|
||||
url: `http://localhost:3005/user/${settings.healthCheckUserId}/check`,
|
||||
json: {
|
||||
words: ['helllo'],
|
||||
language: 'en'
|
||||
},
|
||||
timeout: 1000 * 20
|
||||
}
|
||||
return request.post(opts, function(err, response, body) {
|
||||
if (err != null) {
|
||||
return res.sendStatus(500)
|
||||
}
|
||||
const numberOfSuggestions = __guard__(
|
||||
__guard__(
|
||||
__guard__(body != null ? body.misspellings : undefined, x2 => x2[0]),
|
||||
x1 => x1.suggestions
|
||||
),
|
||||
x => x.length
|
||||
)
|
||||
if (numberOfSuggestions > 10) {
|
||||
logger.log('health check passed')
|
||||
return res.sendStatus(200)
|
||||
} else {
|
||||
logger.err({ body, numberOfSuggestions }, 'health check failed')
|
||||
return res.sendStatus(500)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
function __guard__(value, transform) {
|
||||
return typeof value !== 'undefined' && value !== null
|
||||
? transform(value)
|
||||
: undefined
|
||||
}
|
||||
76
services/spelling/app/js/LearnedWordsManager.js
Normal file
76
services/spelling/app/js/LearnedWordsManager.js
Normal file
|
|
@ -0,0 +1,76 @@
|
|||
// TODO: This file was created by bulk-decaffeinate.
|
||||
// Sanity-check the conversion and remove this comment.
|
||||
/*
|
||||
* decaffeinate suggestions:
|
||||
* DS102: Remove unnecessary code created because of implicit returns
|
||||
* DS207: Consider shorter variations of null checks
|
||||
* Full docs: https://github.com/decaffeinate/decaffeinate/blob/master/docs/suggestions.md
|
||||
*/
|
||||
let LearnedWordsManager
|
||||
const db = require('./DB')
|
||||
const mongoCache = require('./MongoCache')
|
||||
const logger = require('logger-sharelatex')
|
||||
const metrics = require('metrics-sharelatex')
|
||||
|
||||
module.exports = LearnedWordsManager = {
|
||||
learnWord(userToken, word, callback) {
|
||||
if (callback == null) {
|
||||
callback = () => {}
|
||||
}
|
||||
mongoCache.del(userToken)
|
||||
return db.spellingPreferences.update(
|
||||
{
|
||||
token: userToken
|
||||
},
|
||||
{
|
||||
$push: { learnedWords: word }
|
||||
},
|
||||
{
|
||||
upsert: true
|
||||
},
|
||||
callback
|
||||
)
|
||||
},
|
||||
|
||||
getLearnedWords(userToken, callback) {
|
||||
if (callback == null) {
|
||||
callback = () => {}
|
||||
}
|
||||
const mongoCachedWords = mongoCache.get(userToken)
|
||||
if (mongoCachedWords != null) {
|
||||
metrics.inc('mongoCache', 0.1, { status: 'hit' })
|
||||
return callback(null, mongoCachedWords)
|
||||
}
|
||||
|
||||
metrics.inc('mongoCache', 0.1, { status: 'miss' })
|
||||
logger.info({ userToken }, 'mongoCache miss')
|
||||
|
||||
return db.spellingPreferences.findOne({ token: userToken }, function(
|
||||
error,
|
||||
preferences
|
||||
) {
|
||||
if (error != null) {
|
||||
return callback(error)
|
||||
}
|
||||
const words =
|
||||
(preferences != null ? preferences.learnedWords : undefined) || []
|
||||
mongoCache.set(userToken, words)
|
||||
return callback(null, words)
|
||||
})
|
||||
},
|
||||
|
||||
deleteUsersLearnedWords(userToken, callback) {
|
||||
if (callback == null) {
|
||||
callback = () => {}
|
||||
}
|
||||
return db.spellingPreferences.remove({ token: userToken }, callback)
|
||||
}
|
||||
}
|
||||
;['learnWord', 'getLearnedWords'].map(method =>
|
||||
metrics.timeAsyncMethod(
|
||||
LearnedWordsManager,
|
||||
method,
|
||||
'mongo.LearnedWordsManager',
|
||||
logger
|
||||
)
|
||||
)
|
||||
11
services/spelling/app/js/MongoCache.js
Normal file
11
services/spelling/app/js/MongoCache.js
Normal file
|
|
@ -0,0 +1,11 @@
|
|||
// TODO: This file was created by bulk-decaffeinate.
|
||||
// Sanity-check the conversion and remove this comment.
|
||||
const LRU = require('lru-cache')
|
||||
const cacheOpts = {
|
||||
max: 15000,
|
||||
maxAge: 1000 * 60 * 60 * 10
|
||||
}
|
||||
|
||||
const cache = LRU(cacheOpts)
|
||||
|
||||
module.exports = cache
|
||||
111
services/spelling/app/js/SpellingAPIController.js
Normal file
111
services/spelling/app/js/SpellingAPIController.js
Normal file
|
|
@ -0,0 +1,111 @@
|
|||
// TODO: This file was created by bulk-decaffeinate.
|
||||
// Sanity-check the conversion and remove this comment.
|
||||
/*
|
||||
* decaffeinate suggestions:
|
||||
* DS102: Remove unnecessary code created because of implicit returns
|
||||
* DS103: Rewrite code to no longer use __guard__
|
||||
* DS207: Consider shorter variations of null checks
|
||||
* Full docs: https://github.com/decaffeinate/decaffeinate/blob/master/docs/suggestions.md
|
||||
*/
|
||||
const SpellingAPIManager = require('./SpellingAPIManager')
|
||||
const logger = require('logger-sharelatex')
|
||||
const metrics = require('metrics-sharelatex')
|
||||
|
||||
module.exports = {
|
||||
check(req, res, next) {
|
||||
metrics.inc('spelling-check', 0.1)
|
||||
logger.info(
|
||||
{
|
||||
token: __guard__(req != null ? req.params : undefined, x => x.user_id),
|
||||
word_count: __guard__(
|
||||
__guard__(req != null ? req.body : undefined, x2 => x2.words),
|
||||
x1 => x1.length
|
||||
)
|
||||
},
|
||||
'running check'
|
||||
)
|
||||
return SpellingAPIManager.runRequest(req.params.user_id, req.body, function(
|
||||
error,
|
||||
result
|
||||
) {
|
||||
if (error != null) {
|
||||
logger.err(
|
||||
{
|
||||
err: error,
|
||||
user_id: __guard__(
|
||||
req != null ? req.params : undefined,
|
||||
x3 => x3.user_id
|
||||
),
|
||||
word_count: __guard__(
|
||||
__guard__(req != null ? req.body : undefined, x5 => x5.words),
|
||||
x4 => x4.length
|
||||
)
|
||||
},
|
||||
'error processing spelling request'
|
||||
)
|
||||
return res.sendStatus(500)
|
||||
}
|
||||
return res.send(result)
|
||||
})
|
||||
},
|
||||
|
||||
learn(req, res, next) {
|
||||
metrics.inc('spelling-learn', 0.1)
|
||||
logger.info(
|
||||
{
|
||||
token: __guard__(req != null ? req.params : undefined, x => x.user_id),
|
||||
word: __guard__(req != null ? req.body : undefined, x1 => x1.word)
|
||||
},
|
||||
'learning word'
|
||||
)
|
||||
return SpellingAPIManager.learnWord(req.params.user_id, req.body, function(
|
||||
error,
|
||||
result
|
||||
) {
|
||||
if (error != null) {
|
||||
return next(error)
|
||||
}
|
||||
res.sendStatus(200)
|
||||
return next()
|
||||
})
|
||||
},
|
||||
|
||||
deleteDic(req, res, next) {
|
||||
logger.log(
|
||||
{
|
||||
token: __guard__(req != null ? req.params : undefined, x => x.user_id),
|
||||
word: __guard__(req != null ? req.body : undefined, x1 => x1.word)
|
||||
},
|
||||
'deleting user dictionary'
|
||||
)
|
||||
return SpellingAPIManager.deleteDic(req.params.user_id, function(error) {
|
||||
if (error != null) {
|
||||
return next(error)
|
||||
}
|
||||
return res.sendStatus(204)
|
||||
})
|
||||
},
|
||||
|
||||
getDic(req, res, next) {
|
||||
logger.info(
|
||||
{
|
||||
token: __guard__(req != null ? req.params : undefined, x => x.user_id)
|
||||
},
|
||||
'getting user dictionary'
|
||||
)
|
||||
return SpellingAPIManager.getDic(req.params.user_id, function(
|
||||
error,
|
||||
words
|
||||
) {
|
||||
if (error != null) {
|
||||
return next(error)
|
||||
}
|
||||
return res.send(words)
|
||||
})
|
||||
}
|
||||
}
|
||||
function __guard__(value, transform) {
|
||||
return typeof value !== 'undefined' && value !== null
|
||||
? transform(value)
|
||||
: undefined
|
||||
}
|
||||
81
services/spelling/app/js/SpellingAPIManager.js
Normal file
81
services/spelling/app/js/SpellingAPIManager.js
Normal file
|
|
@ -0,0 +1,81 @@
|
|||
// TODO: This file was created by bulk-decaffeinate.
|
||||
// Sanity-check the conversion and remove this comment.
|
||||
/*
|
||||
* decaffeinate suggestions:
|
||||
* DS102: Remove unnecessary code created because of implicit returns
|
||||
* DS207: Consider shorter variations of null checks
|
||||
* Full docs: https://github.com/decaffeinate/decaffeinate/blob/master/docs/suggestions.md
|
||||
*/
|
||||
let SpellingAPIManager
|
||||
const ASpell = require('./ASpell')
|
||||
const LearnedWordsManager = require('./LearnedWordsManager')
|
||||
|
||||
module.exports = SpellingAPIManager = {
|
||||
whitelist: ['ShareLaTeX', 'sharelatex', 'LaTeX', 'http', 'https', 'www'],
|
||||
|
||||
runRequest(token, request, callback) {
|
||||
if (callback == null) {
|
||||
callback = () => {}
|
||||
}
|
||||
if (request.words == null) {
|
||||
return callback(new Error('malformed JSON'))
|
||||
}
|
||||
|
||||
const lang = request.language || 'en'
|
||||
|
||||
const check = (words, callback) =>
|
||||
ASpell.checkWords(lang, words, (error, misspellings) =>
|
||||
callback(error, { misspellings })
|
||||
)
|
||||
const wordsToCheck = request.words || []
|
||||
|
||||
if (token != null) {
|
||||
return LearnedWordsManager.getLearnedWords(token, function(
|
||||
error,
|
||||
learnedWords
|
||||
) {
|
||||
if (error != null) {
|
||||
return callback(error)
|
||||
}
|
||||
const words = wordsToCheck.slice(0, 10000)
|
||||
return check(words, function(error, result) {
|
||||
if (error != null) {
|
||||
return callback(error)
|
||||
}
|
||||
result.misspellings = result.misspellings.filter(function(m) {
|
||||
const word = words[m.index]
|
||||
return (
|
||||
learnedWords.indexOf(word) === -1 &&
|
||||
SpellingAPIManager.whitelist.indexOf(word) === -1
|
||||
)
|
||||
})
|
||||
return callback(error, result)
|
||||
})
|
||||
})
|
||||
} else {
|
||||
return check(wordsToCheck, callback)
|
||||
}
|
||||
},
|
||||
|
||||
learnWord(token, request, callback) {
|
||||
if (callback == null) {
|
||||
callback = () => {}
|
||||
}
|
||||
if (request.word == null) {
|
||||
return callback(new Error('malformed JSON'))
|
||||
}
|
||||
if (token == null) {
|
||||
return callback(new Error('no token provided'))
|
||||
}
|
||||
|
||||
return LearnedWordsManager.learnWord(token, request.word, callback)
|
||||
},
|
||||
|
||||
deleteDic(token, callback) {
|
||||
return LearnedWordsManager.deleteUsersLearnedWords(token, callback)
|
||||
},
|
||||
|
||||
getDic(token, callback) {
|
||||
return LearnedWordsManager.getLearnedWords(token, callback)
|
||||
}
|
||||
}
|
||||
|
|
@ -1,8 +1,8 @@
|
|||
spelling
|
||||
--language=coffeescript
|
||||
--node-version=6.16.0
|
||||
--acceptance-creds=None
|
||||
--dependencies=mongo,redis
|
||||
--docker-repos=gcr.io/overleaf-ops
|
||||
--build-target=docker
|
||||
--script-version=1.1.21
|
||||
--build-target=docker
|
||||
--dependencies=mongo,redis
|
||||
--language=es
|
||||
--docker-repos=gcr.io/overleaf-ops
|
||||
--acceptance-creds=None
|
||||
|
|
|
|||
|
|
@ -1,18 +0,0 @@
|
|||
Path = require('path')
|
||||
|
||||
module.exports = Settings =
|
||||
internal:
|
||||
spelling:
|
||||
port: 3005
|
||||
host: process.env["LISTEN_ADDRESS"] or "localhost"
|
||||
|
||||
mongo:
|
||||
url: process.env['MONGO_CONNECTION_STRING'] or "mongodb://#{process.env["MONGO_HOST"] or "localhost"}/sharelatex"
|
||||
|
||||
cacheDir: Path.resolve "cache"
|
||||
|
||||
|
||||
healthCheckUserId: "53c64d2fd68c8d000010bb5f"
|
||||
|
||||
sentry:
|
||||
dsn: process.env.SENTRY_DSN
|
||||
24
services/spelling/config/settings.defaults.js
Normal file
24
services/spelling/config/settings.defaults.js
Normal file
|
|
@ -0,0 +1,24 @@
|
|||
const Path = require('path')
|
||||
|
||||
module.exports = {
|
||||
internal: {
|
||||
spelling: {
|
||||
port: 3005,
|
||||
host: process.env['LISTEN_ADDRESS'] || 'localhost'
|
||||
}
|
||||
},
|
||||
|
||||
mongo: {
|
||||
url:
|
||||
process.env['MONGO_CONNECTION_STRING'] ||
|
||||
`mongodb://${process.env['MONGO_HOST'] || 'localhost'}/sharelatex`
|
||||
},
|
||||
|
||||
cacheDir: Path.resolve('cache'),
|
||||
|
||||
healthCheckUserId: '53c64d2fd68c8d000010bb5f',
|
||||
|
||||
sentry: {
|
||||
dsn: process.env.SENTRY_DSN
|
||||
}
|
||||
}
|
||||
79
services/spelling/decaffeinate.sh
Normal file
79
services/spelling/decaffeinate.sh
Normal file
|
|
@ -0,0 +1,79 @@
|
|||
#!/usr/bin/env bash
|
||||
set -ex
|
||||
|
||||
# Check .eslintrc and .prettierc are present
|
||||
|
||||
npm install --save-dev eslint eslint-config-prettier eslint-config-standard \
|
||||
eslint-plugin-chai-expect eslint-plugin-chai-friendly eslint-plugin-import \
|
||||
eslint-plugin-mocha eslint-plugin-node eslint-plugin-prettier \
|
||||
eslint-plugin-promise eslint-plugin-standard prettier-eslint-cli
|
||||
|
||||
git add .
|
||||
git commit -m "Decaffeinate: add eslint and prettier rc files"
|
||||
|
||||
|
||||
echo "------------------------"
|
||||
echo "----------APP-----------"
|
||||
echo "------------------------"
|
||||
|
||||
# bulk-decaffeinate will commit for you
|
||||
npx bulk-decaffeinate convert --dir app/coffee
|
||||
npx bulk-decaffeinate clean
|
||||
|
||||
git mv app/coffee app/js
|
||||
git commit -m "Rename app/coffee dir to app/js"
|
||||
|
||||
npx prettier-eslint 'app/js/**/*.js' --write
|
||||
git add .
|
||||
git commit -m "Prettier: convert app/js decaffeinated files to Prettier format"
|
||||
|
||||
|
||||
echo "-------------------------"
|
||||
echo "--------UNIT TESTS-------"
|
||||
echo "-------------------------"
|
||||
npx bulk-decaffeinate convert --dir test/unit/coffee
|
||||
npx bulk-decaffeinate clean
|
||||
|
||||
git mv test/unit/coffee test/unit/js
|
||||
git commit -m "Rename test/unit/coffee to test/unit/js"
|
||||
|
||||
npx prettier-eslint 'test/unit/js/**/*.js' --write
|
||||
git add .
|
||||
git commit -m "Prettier: convert test/unit decaffeinated files to Prettier format"
|
||||
|
||||
|
||||
echo "-------------------------"
|
||||
echo "-------STRESS TESTS------"
|
||||
echo "-------------------------"
|
||||
|
||||
npx bulk-decaffeinate convert --dir test/stress/coffee
|
||||
npx bulk-decaffeinate clean
|
||||
|
||||
git mv test/stress/coffee test/stress/js
|
||||
git commit -m "Rename test/stress/coffee to test/stress/js"
|
||||
|
||||
npx prettier-eslint 'test/stress/js/**/*.js' --write
|
||||
git add .
|
||||
git commit -m "Prettier: convert test/stress decaffeinated files to Prettier format"
|
||||
|
||||
|
||||
echo "--------------------------"
|
||||
echo "-----INDIVIDUAL FILES-----"
|
||||
echo "--------------------------"
|
||||
|
||||
rm -f app.js config/settings.defaults.js
|
||||
git mv app.coffee app.js
|
||||
git mv config/settings.defaults.coffee config/settings.defaults.js
|
||||
git commit -m "Rename individual coffee files to js files"
|
||||
|
||||
decaffeinate app.js
|
||||
decaffeinate config/settings.defaults.js
|
||||
git add .
|
||||
git commit -m "Decaffeinate: convert individual files to js"
|
||||
|
||||
npx prettier-eslint 'app.js' 'config/settings.defaults.js' --write
|
||||
|
||||
git add .
|
||||
git commit -m "Prettier: convert individual decaffeinated files to Prettier format"
|
||||
|
||||
echo "done."
|
||||
|
|
@ -10,10 +10,9 @@
|
|||
},
|
||||
|
||||
"watch": [
|
||||
"app/coffee/",
|
||||
"app.coffee",
|
||||
"app/js/",
|
||||
"app.js",
|
||||
"config/"
|
||||
],
|
||||
"ext": "coffee"
|
||||
|
||||
"ext": "js"
|
||||
}
|
||||
|
|
|
|||
2
services/spelling/npm-shrinkwrap.json
generated
2
services/spelling/npm-shrinkwrap.json
generated
|
|
@ -1849,4 +1849,4 @@
|
|||
"resolved": "https://registry.npmjs.org/yallist/-/yallist-2.1.2.tgz"
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -8,16 +8,19 @@
|
|||
},
|
||||
"scripts": {
|
||||
"compile:app": "([ -e app/coffee ] && coffee -m $COFFEE_OPTIONS -o app/js -c app/coffee || echo 'No CoffeeScript folder to compile') && ( [ -e app.coffee ] && coffee -m $COFFEE_OPTIONS -c app.coffee || echo 'No CoffeeScript app to compile')",
|
||||
"start": "npm run compile:app && node $NODE_APP_OPTIONS app.js",
|
||||
"test:acceptance:_run": "mocha --recursive --reporter spec --timeout 30000 --exit $@ test/acceptance/js",
|
||||
"test:acceptance": "npm run compile:app && npm run compile:acceptance_tests && npm run test:acceptance:_run -- --grep=$MOCHA_GREP",
|
||||
"start": "node $NODE_APP_OPTIONS app.js",
|
||||
"test:acceptance:_run": "mocha --recursive --reporter spec --timeout 15000 --exit $@ test/acceptance/js",
|
||||
"test:acceptance": "npm run test:acceptance:_run -- --grep=$MOCHA_GREP",
|
||||
"test:unit:_run": "mocha --recursive --reporter spec --exit $@ test/unit/js",
|
||||
"test:unit": "npm run compile:app && npm run compile:unit_tests && npm run test:unit:_run -- --grep=$MOCHA_GREP",
|
||||
"test:unit": "npm run test:unit:_run -- --grep=$MOCHA_GREP",
|
||||
"compile:unit_tests": "[ ! -e test/unit/coffee ] && echo 'No unit tests to compile' || coffee -o test/unit/js -c test/unit/coffee",
|
||||
"compile:acceptance_tests": "[ ! -e test/acceptance/coffee ] && echo 'No acceptance tests to compile' || coffee -o test/acceptance/js -c test/acceptance/coffee",
|
||||
"compile:all": "npm run compile:app && npm run compile:unit_tests && npm run compile:acceptance_tests && npm run compile:smoke_tests",
|
||||
"nodemon": "nodemon --config nodemon.json",
|
||||
"compile:smoke_tests": "[ ! -e test/smoke/coffee ] && echo 'No smoke tests to compile' || coffee -o test/smoke/js -c test/smoke/coffee"
|
||||
"compile:smoke_tests": "[ ! -e test/smoke/coffee ] && echo 'No smoke tests to compile' || coffee -o test/smoke/js -c test/smoke/coffee",
|
||||
"lint": "node_modules/.bin/eslint .",
|
||||
"format": "node_modules/.bin/prettier-eslint '**/*.js' --list-different",
|
||||
"format:fix": "node_modules/.bin/prettier-eslint '**/*.js' --write"
|
||||
},
|
||||
"version": "0.1.4",
|
||||
"dependencies": {
|
||||
|
|
@ -39,7 +42,19 @@
|
|||
"devDependencies": {
|
||||
"bunyan": "^1.0.0",
|
||||
"chai": "",
|
||||
"eslint": "^6.0.1",
|
||||
"eslint-config-prettier": "^6.0.0",
|
||||
"eslint-config-standard": "^12.0.0",
|
||||
"eslint-plugin-chai-expect": "^2.0.1",
|
||||
"eslint-plugin-chai-friendly": "^0.4.1",
|
||||
"eslint-plugin-import": "^2.18.0",
|
||||
"eslint-plugin-mocha": "^5.3.0",
|
||||
"eslint-plugin-node": "^9.1.0",
|
||||
"eslint-plugin-prettier": "^3.1.0",
|
||||
"eslint-plugin-promise": "^4.2.1",
|
||||
"eslint-plugin-standard": "^4.0.0",
|
||||
"mocha": "^4.1.0",
|
||||
"prettier-eslint-cli": "^5.0.0",
|
||||
"sandboxed-module": "",
|
||||
"sinon": "^1.17.0"
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1,86 +0,0 @@
|
|||
# N requests in parallel
|
||||
# send P correct words and Q incorrect words
|
||||
# generate incorrect words by qq+random
|
||||
|
||||
async = require "async"
|
||||
request = require "request"
|
||||
fs = require "fs"
|
||||
|
||||
# created with
|
||||
# aspell -d en dump master | aspell -l en expand | shuf -n 150000 > words.txt
|
||||
WORDS = "words.txt"
|
||||
wordlist = fs.readFileSync(WORDS).toString().split('\n').filter (w) ->
|
||||
w.match(/^[a-z]+$/)
|
||||
|
||||
generateCorrectWords = (n) ->
|
||||
words = []
|
||||
N = if Math.random() > 0.5 then wordlist.length else 10
|
||||
for i in [1 .. n]
|
||||
j = Math.floor(N * Math.random())
|
||||
words.push wordlist[j]
|
||||
return words
|
||||
|
||||
generateIncorrectWords = (n) ->
|
||||
words = []
|
||||
N = wordlist.length
|
||||
for i in [1 .. n]
|
||||
j = Math.floor(N * Math.random())
|
||||
words.push("qzxq" + wordlist[j])
|
||||
return words
|
||||
|
||||
make_request = (correctWords, incorrectWords, callback) ->
|
||||
correctSet = generateCorrectWords(correctWords)
|
||||
incorrectSet = generateIncorrectWords(incorrectWords)
|
||||
correctSet.push('constructor')
|
||||
incorrectSet.push('qzxqfoofoofoo')
|
||||
full = correctSet.concat incorrectSet
|
||||
bad = []
|
||||
for w, i in correctSet
|
||||
bad[i] = false
|
||||
for w, i in incorrectSet
|
||||
bad[i+correctSet.length] = true
|
||||
k = full.length
|
||||
full.forEach (e, i) ->
|
||||
j = Math.floor(k * Math.random())
|
||||
[ full[i], full[j] ] = [ full[j], full[i] ]
|
||||
[ bad[i], bad[j] ] = [ bad[j], bad[i] ]
|
||||
expected = []
|
||||
for tf, i in bad
|
||||
if tf
|
||||
expected.push {index: i, word: full[i]}
|
||||
request.post 'http://localhost:3005/user/1/check', json:true, body: {words: full}, (err, req, body) ->
|
||||
misspellings = body.misspellings
|
||||
console.log JSON.stringify({full: full, misspellings: misspellings})
|
||||
if expected.length != misspellings.length
|
||||
console.log "ERROR: length mismatch", expected.length, misspellings.length
|
||||
console.log full, bad
|
||||
console.log 'expected', expected, 'mispellings', misspellings
|
||||
for i in [0 .. Math.max(expected.length, misspellings.length)-1]
|
||||
if expected[i].index != misspellings[i].index
|
||||
console.log "ERROR", i, expected[i], misspellings[i], full[misspellings[i].index]
|
||||
for m in misspellings
|
||||
console.log full[m.index], "=>", m
|
||||
process.exit()
|
||||
callback("error")
|
||||
else
|
||||
for m,i in body.misspellings
|
||||
if m.index != expected[i].index
|
||||
console.log "ERROR AT RESULT", i, m, expected[i]
|
||||
process.exit()
|
||||
callback("error")
|
||||
callback(null, full)
|
||||
|
||||
q = async.queue (task, callback) ->
|
||||
setTimeout () ->
|
||||
make_request task.correct, task.incorrect, callback
|
||||
, Math.random() * 100
|
||||
, 3
|
||||
|
||||
q.drain = () ->
|
||||
console.log('all items have been processed');
|
||||
|
||||
for i in [0 .. 1000]
|
||||
q.push({correct: Math.floor(30*Math.random()) + 1, incorrect: Math.floor(3*Math.random())})
|
||||
# if Math.random() < 0.1
|
||||
# else
|
||||
# q.push({correct: Math.floor(100*Math.random()) + 1, incorrect: Math.floor(3*Math.random())})
|
||||
162
services/spelling/test/stress/js/stressTest.js
Normal file
162
services/spelling/test/stress/js/stressTest.js
Normal file
|
|
@ -0,0 +1,162 @@
|
|||
/* eslint-disable */
|
||||
// TODO: This file was created by bulk-decaffeinate.
|
||||
// Sanity-check the conversion and remove this comment.
|
||||
/*
|
||||
* decaffeinate suggestions:
|
||||
* DS101: Remove unnecessary use of Array.from
|
||||
* DS102: Remove unnecessary code created because of implicit returns
|
||||
* DS104: Avoid inline assignments
|
||||
* DS202: Simplify dynamic range loops
|
||||
* Full docs: https://github.com/decaffeinate/decaffeinate/blob/master/docs/suggestions.md
|
||||
*/
|
||||
// N requests in parallel
|
||||
// send P correct words and Q incorrect words
|
||||
// generate incorrect words by qq+random
|
||||
|
||||
const async = require('async')
|
||||
const request = require('request')
|
||||
const fs = require('fs')
|
||||
|
||||
// created with
|
||||
// aspell -d en dump master | aspell -l en expand | shuf -n 150000 > words.txt
|
||||
const WORDS = 'words.txt'
|
||||
const wordlist = fs
|
||||
.readFileSync(WORDS)
|
||||
.toString()
|
||||
.split('\n')
|
||||
.filter(w => w.match(/^[a-z]+$/))
|
||||
|
||||
const generateCorrectWords = function(n) {
|
||||
const words = []
|
||||
const N = Math.random() > 0.5 ? wordlist.length : 10
|
||||
for (
|
||||
let i = 1, end = n, asc = 1 <= end;
|
||||
asc ? i <= end : i >= end;
|
||||
asc ? i++ : i--
|
||||
) {
|
||||
const j = Math.floor(N * Math.random())
|
||||
words.push(wordlist[j])
|
||||
}
|
||||
return words
|
||||
}
|
||||
|
||||
const generateIncorrectWords = function(n) {
|
||||
const words = []
|
||||
const N = wordlist.length
|
||||
for (
|
||||
let i = 1, end = n, asc = 1 <= end;
|
||||
asc ? i <= end : i >= end;
|
||||
asc ? i++ : i--
|
||||
) {
|
||||
const j = Math.floor(N * Math.random())
|
||||
words.push(`qzxq${wordlist[j]}`)
|
||||
}
|
||||
return words
|
||||
}
|
||||
|
||||
const make_request = function(correctWords, incorrectWords, callback) {
|
||||
let i, j, w
|
||||
let i1
|
||||
let j1
|
||||
const correctSet = generateCorrectWords(correctWords)
|
||||
const incorrectSet = generateIncorrectWords(incorrectWords)
|
||||
correctSet.push('constructor')
|
||||
incorrectSet.push('qzxqfoofoofoo')
|
||||
const full = correctSet.concat(incorrectSet)
|
||||
const bad = []
|
||||
for (j = 0, i = j; j < correctSet.length; j++, i = j) {
|
||||
w = correctSet[i]
|
||||
bad[i] = false
|
||||
}
|
||||
for (i1 = 0, i = i1; i1 < incorrectSet.length; i1++, i = i1) {
|
||||
w = incorrectSet[i]
|
||||
bad[i + correctSet.length] = true
|
||||
}
|
||||
const k = full.length
|
||||
full.forEach(function(e, i) {
|
||||
let ref
|
||||
j = Math.floor(k * Math.random())
|
||||
;[full[i], full[j]] = Array.from([full[j], full[i]])
|
||||
return ([bad[i], bad[j]] = Array.from((ref = [bad[j], bad[i]]))), ref
|
||||
})
|
||||
const expected = []
|
||||
for (j1 = 0, i = j1; j1 < bad.length; j1++, i = j1) {
|
||||
const tf = bad[i]
|
||||
if (tf) {
|
||||
expected.push({ index: i, word: full[i] })
|
||||
}
|
||||
}
|
||||
return request.post(
|
||||
'http://localhost:3005/user/1/check',
|
||||
{ json: true, body: { words: full } },
|
||||
function(err, req, body) {
|
||||
let m
|
||||
const { misspellings } = body
|
||||
console.log(JSON.stringify({ full, misspellings }))
|
||||
if (expected.length !== misspellings.length) {
|
||||
let asc, end
|
||||
console.log(
|
||||
'ERROR: length mismatch',
|
||||
expected.length,
|
||||
misspellings.length
|
||||
)
|
||||
console.log(full, bad)
|
||||
console.log('expected', expected, 'mispellings', misspellings)
|
||||
for (
|
||||
i = 0,
|
||||
end = Math.max(expected.length, misspellings.length) - 1,
|
||||
asc = 0 <= end;
|
||||
asc ? i <= end : i >= end;
|
||||
asc ? i++ : i--
|
||||
) {
|
||||
if (expected[i].index !== misspellings[i].index) {
|
||||
console.log(
|
||||
'ERROR',
|
||||
i,
|
||||
expected[i],
|
||||
misspellings[i],
|
||||
full[misspellings[i].index]
|
||||
)
|
||||
}
|
||||
}
|
||||
for (m of Array.from(misspellings)) {
|
||||
console.log(full[m.index], '=>', m)
|
||||
}
|
||||
process.exit()
|
||||
callback('error')
|
||||
} else {
|
||||
for (i = 0; i < body.misspellings.length; i++) {
|
||||
m = body.misspellings[i]
|
||||
if (m.index !== expected[i].index) {
|
||||
console.log('ERROR AT RESULT', i, m, expected[i])
|
||||
process.exit()
|
||||
callback('error')
|
||||
}
|
||||
}
|
||||
}
|
||||
return callback(null, full)
|
||||
}
|
||||
)
|
||||
}
|
||||
|
||||
const q = async.queue(
|
||||
(task, callback) =>
|
||||
setTimeout(
|
||||
() => make_request(task.correct, task.incorrect, callback),
|
||||
Math.random() * 100
|
||||
),
|
||||
|
||||
3
|
||||
)
|
||||
|
||||
q.drain = () => console.log('all items have been processed')
|
||||
|
||||
for (let i = 0; i <= 1000; i++) {
|
||||
q.push({
|
||||
correct: Math.floor(30 * Math.random()) + 1,
|
||||
incorrect: Math.floor(3 * Math.random())
|
||||
})
|
||||
}
|
||||
// if Math.random() < 0.1
|
||||
// else
|
||||
// q.push({correct: Math.floor(100*Math.random()) + 1, incorrect: Math.floor(3*Math.random())})
|
||||
|
|
@ -1,70 +0,0 @@
|
|||
sinon = require 'sinon'
|
||||
chai = require 'chai'
|
||||
should = chai.should()
|
||||
SandboxedModule = require('sandboxed-module')
|
||||
assert = require("chai").assert
|
||||
|
||||
describe "ASpell", ->
|
||||
beforeEach ->
|
||||
@ASpell = SandboxedModule.require "../../../app/js/ASpell", requires:
|
||||
"logger-sharelatex":
|
||||
log:->
|
||||
info:->
|
||||
err:->
|
||||
'metrics-sharelatex':
|
||||
gauge:->
|
||||
inc: ->
|
||||
|
||||
describe "a correctly spelled word", ->
|
||||
beforeEach (done) ->
|
||||
@ASpell.checkWords "en", ["word"], (error, @result) => done()
|
||||
|
||||
it "should not correct the word", ->
|
||||
@result.length.should.equal 0
|
||||
|
||||
describe "a misspelled word", ->
|
||||
beforeEach (done) ->
|
||||
@ASpell.checkWords "en", ["bussines"], (error, @result) => done()
|
||||
|
||||
it "should correct the word", ->
|
||||
@result.length.should.equal 1
|
||||
@result[0].suggestions.indexOf("business").should.not.equal -1
|
||||
|
||||
describe "multiple words", ->
|
||||
beforeEach (done) ->
|
||||
@ASpell.checkWords "en", ["bussines", "word", "neccesary"], (error, @result) => done()
|
||||
|
||||
it "should correct the incorrect words", ->
|
||||
@result[0].index.should.equal 0
|
||||
@result[0].suggestions.indexOf("business").should.not.equal -1
|
||||
@result[1].index.should.equal 2
|
||||
@result[1].suggestions.indexOf("necessary").should.not.equal -1
|
||||
|
||||
describe "without a valid language", ->
|
||||
beforeEach (done) ->
|
||||
@ASpell.checkWords "notALang", ["banana"], (@error, @result) => done()
|
||||
|
||||
it "should return an error", ->
|
||||
should.exist @error
|
||||
|
||||
describe "when there are no suggestions", ->
|
||||
beforeEach (done) ->
|
||||
@ASpell.checkWords "en", ["asdkfjalkdjfadhfkajsdhfashdfjhadflkjadhflajsd"], (@error, @result) => done()
|
||||
|
||||
it "should return a blank array", ->
|
||||
@result.length.should.equal 1
|
||||
assert.deepEqual @result[0].suggestions, []
|
||||
|
||||
describe "when the request times out", ->
|
||||
beforeEach (done) ->
|
||||
words = ("abcdefg" for i in [0..1000])
|
||||
@ASpell.ASPELL_TIMEOUT = 1
|
||||
@start = Date.now()
|
||||
@ASpell.checkWords "en", words, (error, @result) => done()
|
||||
|
||||
# Note that this test fails on OS X, due to differing pipe behaviour
|
||||
# on killing the child process. It can be tested successfully on Travis
|
||||
# or the CI server.
|
||||
it "should return in reasonable time", () ->
|
||||
delta = Date.now()-@start
|
||||
delta.should.be.below(@ASpell.ASPELL_TIMEOUT + 1000)
|
||||
|
|
@ -1,97 +0,0 @@
|
|||
sinon = require('sinon')
|
||||
chai = require 'chai'
|
||||
expect = chai.expect
|
||||
SandboxedModule = require('sandboxed-module')
|
||||
modulePath = require('path').join __dirname, '../../../app/js/LearnedWordsManager'
|
||||
assert = require("chai").assert
|
||||
should = require("chai").should()
|
||||
describe "LearnedWordsManager", ->
|
||||
beforeEach ->
|
||||
@token = "a6b3cd919ge"
|
||||
@callback = sinon.stub()
|
||||
@db =
|
||||
spellingPreferences:
|
||||
update: sinon.stub().callsArg(3)
|
||||
@cache =
|
||||
get:sinon.stub()
|
||||
set:sinon.stub()
|
||||
del:sinon.stub()
|
||||
@LearnedWordsManager = SandboxedModule.require modulePath, requires:
|
||||
"./DB" : @db
|
||||
"./MongoCache":@cache
|
||||
"logger-sharelatex":
|
||||
log:->
|
||||
err:->
|
||||
info:->
|
||||
'metrics-sharelatex': {timeAsyncMethod: sinon.stub(), inc: sinon.stub()}
|
||||
|
||||
describe "learnWord", ->
|
||||
beforeEach ->
|
||||
@word = "instanton"
|
||||
@LearnedWordsManager.learnWord @token, @word, @callback
|
||||
|
||||
it "should insert the word in the word list in the database", ->
|
||||
expect(
|
||||
@db.spellingPreferences.update.calledWith({
|
||||
token: @token
|
||||
}, {
|
||||
$push : learnedWords: @word
|
||||
}, {
|
||||
upsert: true
|
||||
})
|
||||
).to.equal true
|
||||
|
||||
it "should call the callback", ->
|
||||
expect(@callback.called).to.equal true
|
||||
|
||||
describe "getLearnedWords", ->
|
||||
beforeEach ->
|
||||
@wordList = ["apples", "bananas", "pears"]
|
||||
@db.spellingPreferences.findOne = (conditions, callback) =>
|
||||
callback null, learnedWords: @wordList
|
||||
sinon.spy @db.spellingPreferences, "findOne"
|
||||
@LearnedWordsManager.getLearnedWords @token, @callback
|
||||
|
||||
it "should get the word list for the given user", ->
|
||||
expect(
|
||||
@db.spellingPreferences.findOne.calledWith token: @token
|
||||
).to.equal true
|
||||
|
||||
it "should return the word list in the callback", ->
|
||||
expect(@callback.calledWith null, @wordList).to.equal true
|
||||
|
||||
|
||||
describe "caching the result", ->
|
||||
it 'should use the cache first if it is primed', (done)->
|
||||
@wordList = ["apples", "bananas", "pears"]
|
||||
@cache.get.returns(@wordList)
|
||||
@db.spellingPreferences.findOne = sinon.stub()
|
||||
@LearnedWordsManager.getLearnedWords @token, (err, spellings)=>
|
||||
@db.spellingPreferences.findOne.called.should.equal false
|
||||
assert.deepEqual @wordList, spellings
|
||||
done()
|
||||
|
||||
it 'should set the cache after hitting the db', (done)->
|
||||
@wordList = ["apples", "bananas", "pears"]
|
||||
@db.spellingPreferences.findOne = sinon.stub().callsArgWith(1, null, learnedWords: @wordList)
|
||||
@LearnedWordsManager.getLearnedWords @token, (err, spellings)=>
|
||||
@cache.set.calledWith(@token, @wordList).should.equal true
|
||||
done()
|
||||
|
||||
it 'should break cache when update is called', (done)->
|
||||
@word = "instanton"
|
||||
@LearnedWordsManager.learnWord @token, @word, =>
|
||||
@cache.del.calledWith(@token).should.equal true
|
||||
done()
|
||||
|
||||
|
||||
describe "deleteUsersLearnedWords", ->
|
||||
beforeEach ->
|
||||
@db.spellingPreferences.remove = sinon.stub().callsArgWith(1)
|
||||
|
||||
|
||||
it "should get the word list for the given user", (done)->
|
||||
@LearnedWordsManager.deleteUsersLearnedWords @token, =>
|
||||
@db.spellingPreferences.remove.calledWith(token: @token).should.equal true
|
||||
done()
|
||||
|
||||
|
|
@ -1,119 +0,0 @@
|
|||
sinon = require('sinon')
|
||||
chai = require 'chai'
|
||||
expect = chai.expect
|
||||
chai.should()
|
||||
SandboxedModule = require('sandboxed-module')
|
||||
modulePath = require('path').join __dirname, '../../../app/js/SpellingAPIManager'
|
||||
|
||||
describe "SpellingAPIManager", ->
|
||||
beforeEach ->
|
||||
@token = "user-id-123"
|
||||
@ASpell = {}
|
||||
@learnedWords = ["lerned"]
|
||||
@LearnedWordsManager =
|
||||
getLearnedWords: sinon.stub().callsArgWith(1, null, @learnedWords)
|
||||
learnWord: sinon.stub().callsArg(2)
|
||||
|
||||
@SpellingAPIManager = SandboxedModule.require modulePath, requires:
|
||||
"./ASpell" : @ASpell
|
||||
"./LearnedWordsManager" : @LearnedWordsManager
|
||||
|
||||
describe "runRequest", ->
|
||||
beforeEach ->
|
||||
@nonLearnedWords = ["some", "words", "htat", "are", "speled", "rong", "lerned"]
|
||||
@allWords = @nonLearnedWords.concat(@learnedWords)
|
||||
@misspellings = [
|
||||
{ index: 2, suggestions: ["that"] }
|
||||
{ index: 4, suggestions: ["spelled"] }
|
||||
{ index: 5, suggestions: ["wrong", "ring"] }
|
||||
{ index: 6, suggestions: ["learned"] }
|
||||
]
|
||||
@misspellingsWithoutLearnedWords = @misspellings.slice(0,3)
|
||||
|
||||
@ASpell.checkWords = (lang, word, callback) =>
|
||||
callback null, @misspellings
|
||||
sinon.spy @ASpell, "checkWords"
|
||||
|
||||
describe "with sensible JSON", ->
|
||||
beforeEach (done) ->
|
||||
@SpellingAPIManager.runRequest @token, words: @allWords, (error, @result) => done()
|
||||
|
||||
it "should return the words that are spelled incorrectly and not learned", ->
|
||||
expect(@result.misspellings).to.deep.equal @misspellingsWithoutLearnedWords
|
||||
|
||||
describe "with a missing words array", ->
|
||||
beforeEach (done) ->
|
||||
@SpellingAPIManager.runRequest @token, {}, (@error, @result) => done()
|
||||
|
||||
it "should return an error", ->
|
||||
expect(@error).to.exist
|
||||
expect(@error).to.be.instanceof Error
|
||||
expect(@error.message).to.equal "malformed JSON"
|
||||
|
||||
describe "with a missing token", ->
|
||||
beforeEach (done) ->
|
||||
@SpellingAPIManager.runRequest null, words: @allWords, (@error, @result) => done()
|
||||
|
||||
it "should spell check without using any learned words", ->
|
||||
@LearnedWordsManager.getLearnedWords.called.should.equal false
|
||||
|
||||
describe "without a language", ->
|
||||
beforeEach (done) ->
|
||||
@SpellingAPIManager.runRequest @token, words: @allWords, (error, @result) => done()
|
||||
|
||||
it "should use en as the default", ->
|
||||
@ASpell.checkWords.calledWith("en").should.equal true
|
||||
|
||||
describe "with a language", ->
|
||||
beforeEach (done) ->
|
||||
@SpellingAPIManager.runRequest @token, {
|
||||
words: @allWords
|
||||
language: @language = "fr"
|
||||
}, (error, @result) => done()
|
||||
|
||||
it "should use the language", ->
|
||||
@ASpell.checkWords.calledWith(@language).should.equal true
|
||||
|
||||
describe "with a very large collection of words", ->
|
||||
beforeEach (done) ->
|
||||
@manyWords = ("word" for i in [1..100000])
|
||||
@SpellingAPIManager.runRequest @token, words: @manyWords, (error, @result) => done()
|
||||
|
||||
it "should truncate to 10,000 words", ->
|
||||
@ASpell.checkWords.calledWith(sinon.match.any, @manyWords.slice(0, 10000)).should.equal true
|
||||
|
||||
describe 'with words from the whitelist', ->
|
||||
beforeEach (done) ->
|
||||
@whitelistWord = @SpellingAPIManager.whitelist[0]
|
||||
@words = ["One", "Two", @whitelistWord]
|
||||
@SpellingAPIManager.runRequest @token, words: @words, (error, @result) => done()
|
||||
|
||||
it 'should ignore the white-listed word', ->
|
||||
expect(@result.misspellings.length).to.equal @misspellings.length-1
|
||||
|
||||
describe "learnWord", ->
|
||||
describe "without a token", ->
|
||||
beforeEach (done) -> @SpellingAPIManager.learnWord null, word: "banana", (@error) => done()
|
||||
|
||||
it "should return an error", ->
|
||||
expect(@error).to.exist
|
||||
expect(@error).to.be.instanceof Error
|
||||
expect(@error.message).to.equal "no token provided"
|
||||
|
||||
describe "without a word", ->
|
||||
beforeEach (done) -> @SpellingAPIManager.learnWord @token, {}, (@error) => done()
|
||||
|
||||
it "should return an error", ->
|
||||
expect(@error).to.exist
|
||||
expect(@error).to.be.instanceof Error
|
||||
expect(@error.message).to.equal "malformed JSON"
|
||||
|
||||
describe "with a word and a token", ->
|
||||
beforeEach (done) ->
|
||||
@word = "banana"
|
||||
@SpellingAPIManager.learnWord @token, word: @word, (@error) => done()
|
||||
|
||||
it "should call LearnedWordsManager.learnWord", ->
|
||||
@LearnedWordsManager.learnWord.calledWith(@token, @word).should.equal true
|
||||
|
||||
|
||||
145
services/spelling/test/unit/js/ASpellTests.js
Normal file
145
services/spelling/test/unit/js/ASpellTests.js
Normal file
|
|
@ -0,0 +1,145 @@
|
|||
/* eslint-disable
|
||||
handle-callback-err,
|
||||
no-undef
|
||||
*/
|
||||
// TODO: This file was created by bulk-decaffeinate.
|
||||
// Sanity-check the conversion and remove this comment.
|
||||
/*
|
||||
* decaffeinate suggestions:
|
||||
* DS102: Remove unnecessary code created because of implicit returns
|
||||
* Full docs: https://github.com/decaffeinate/decaffeinate/blob/master/docs/suggestions.md
|
||||
*/
|
||||
const chai = require('chai')
|
||||
const should = chai.should()
|
||||
const SandboxedModule = require('sandboxed-module')
|
||||
const { assert } = require('chai')
|
||||
|
||||
describe('ASpell', function() {
|
||||
beforeEach(function() {
|
||||
return (this.ASpell = SandboxedModule.require('../../../app/js/ASpell', {
|
||||
requires: {
|
||||
'logger-sharelatex': {
|
||||
log() {},
|
||||
info() {},
|
||||
err() {}
|
||||
},
|
||||
'metrics-sharelatex': {
|
||||
gauge() {},
|
||||
inc() {}
|
||||
}
|
||||
}
|
||||
}))
|
||||
})
|
||||
|
||||
describe('a correctly spelled word', function() {
|
||||
beforeEach(function(done) {
|
||||
return this.ASpell.checkWords('en', ['word'], (error, result) => {
|
||||
this.result = result
|
||||
return done()
|
||||
})
|
||||
})
|
||||
|
||||
return it('should not correct the word', function() {
|
||||
return this.result.length.should.equal(0)
|
||||
})
|
||||
})
|
||||
|
||||
describe('a misspelled word', function() {
|
||||
beforeEach(function(done) {
|
||||
return this.ASpell.checkWords('en', ['bussines'], (error, result) => {
|
||||
this.result = result
|
||||
return done()
|
||||
})
|
||||
})
|
||||
|
||||
return it('should correct the word', function() {
|
||||
this.result.length.should.equal(1)
|
||||
return this.result[0].suggestions.indexOf('business').should.not.equal(-1)
|
||||
})
|
||||
})
|
||||
|
||||
describe('multiple words', function() {
|
||||
beforeEach(function(done) {
|
||||
return this.ASpell.checkWords(
|
||||
'en',
|
||||
['bussines', 'word', 'neccesary'],
|
||||
(error, result) => {
|
||||
this.result = result
|
||||
return done()
|
||||
}
|
||||
)
|
||||
})
|
||||
|
||||
return it('should correct the incorrect words', function() {
|
||||
this.result[0].index.should.equal(0)
|
||||
this.result[0].suggestions.indexOf('business').should.not.equal(-1)
|
||||
this.result[1].index.should.equal(2)
|
||||
return this.result[1].suggestions
|
||||
.indexOf('necessary')
|
||||
.should.not.equal(-1)
|
||||
})
|
||||
})
|
||||
|
||||
describe('without a valid language', function() {
|
||||
beforeEach(function(done) {
|
||||
return this.ASpell.checkWords('notALang', ['banana'], (error, result) => {
|
||||
this.error = error
|
||||
this.result = result
|
||||
return done()
|
||||
})
|
||||
})
|
||||
|
||||
return it('should return an error', function() {
|
||||
return should.exist(this.error)
|
||||
})
|
||||
})
|
||||
|
||||
describe('when there are no suggestions', function() {
|
||||
beforeEach(function(done) {
|
||||
return this.ASpell.checkWords(
|
||||
'en',
|
||||
['asdkfjalkdjfadhfkajsdhfashdfjhadflkjadhflajsd'],
|
||||
(error, result) => {
|
||||
this.error = error
|
||||
this.result = result
|
||||
return done()
|
||||
}
|
||||
)
|
||||
})
|
||||
|
||||
return it('should return a blank array', function() {
|
||||
this.result.length.should.equal(1)
|
||||
return assert.deepEqual(this.result[0].suggestions, [])
|
||||
})
|
||||
})
|
||||
|
||||
return describe('when the request times out', function() {
|
||||
beforeEach(function(done) {
|
||||
const words = __range__(0, 1000, true).map(i => 'abcdefg')
|
||||
this.ASpell.ASPELL_TIMEOUT = 1
|
||||
this.start = Date.now()
|
||||
return this.ASpell.checkWords('en', words, (error, result) => {
|
||||
this.result = result
|
||||
return done()
|
||||
})
|
||||
})
|
||||
|
||||
// Note that this test fails on OS X, due to differing pipe behaviour
|
||||
// on killing the child process. It can be tested successfully on Travis
|
||||
// or the CI server.
|
||||
return it('should return in reasonable time', function() {
|
||||
const delta = Date.now() - this.start
|
||||
return delta.should.be.below(this.ASpell.ASPELL_TIMEOUT + 1000)
|
||||
})
|
||||
})
|
||||
})
|
||||
|
||||
function __range__(left, right, inclusive) {
|
||||
let range = []
|
||||
let ascending = left < right
|
||||
let end = !inclusive ? right : ascending ? right + 1 : right - 1
|
||||
for (let i = left; ascending ? i < end : i > end; ascending ? i++ : i--) {
|
||||
range.push(i)
|
||||
}
|
||||
return range
|
||||
}
|
||||
163
services/spelling/test/unit/js/LearnedWordsManagerTests.js
Normal file
163
services/spelling/test/unit/js/LearnedWordsManagerTests.js
Normal file
|
|
@ -0,0 +1,163 @@
|
|||
/* eslint-disable
|
||||
handle-callback-err,
|
||||
no-undef
|
||||
*/
|
||||
// TODO: This file was created by bulk-decaffeinate.
|
||||
// Sanity-check the conversion and remove this comment.
|
||||
/*
|
||||
* decaffeinate suggestions:
|
||||
* DS102: Remove unnecessary code created because of implicit returns
|
||||
* Full docs: https://github.com/decaffeinate/decaffeinate/blob/master/docs/suggestions.md
|
||||
*/
|
||||
const sinon = require('sinon')
|
||||
const chai = require('chai')
|
||||
const { expect } = chai
|
||||
const SandboxedModule = require('sandboxed-module')
|
||||
const modulePath = require('path').join(
|
||||
__dirname,
|
||||
'../../../app/js/LearnedWordsManager'
|
||||
)
|
||||
const { assert } = require('chai')
|
||||
describe('LearnedWordsManager', function() {
|
||||
beforeEach(function() {
|
||||
this.token = 'a6b3cd919ge'
|
||||
this.callback = sinon.stub()
|
||||
this.db = {
|
||||
spellingPreferences: {
|
||||
update: sinon.stub().callsArg(3)
|
||||
}
|
||||
}
|
||||
this.cache = {
|
||||
get: sinon.stub(),
|
||||
set: sinon.stub(),
|
||||
del: sinon.stub()
|
||||
}
|
||||
return (this.LearnedWordsManager = SandboxedModule.require(modulePath, {
|
||||
requires: {
|
||||
'./DB': this.db,
|
||||
'./MongoCache': this.cache,
|
||||
'logger-sharelatex': {
|
||||
log() {},
|
||||
err() {},
|
||||
info() {}
|
||||
},
|
||||
'metrics-sharelatex': {
|
||||
timeAsyncMethod: sinon.stub(),
|
||||
inc: sinon.stub()
|
||||
}
|
||||
}
|
||||
}))
|
||||
})
|
||||
|
||||
describe('learnWord', function() {
|
||||
beforeEach(function() {
|
||||
this.word = 'instanton'
|
||||
return this.LearnedWordsManager.learnWord(
|
||||
this.token,
|
||||
this.word,
|
||||
this.callback
|
||||
)
|
||||
})
|
||||
|
||||
it('should insert the word in the word list in the database', function() {
|
||||
return expect(
|
||||
this.db.spellingPreferences.update.calledWith(
|
||||
{
|
||||
token: this.token
|
||||
},
|
||||
{
|
||||
$push: { learnedWords: this.word }
|
||||
},
|
||||
{
|
||||
upsert: true
|
||||
}
|
||||
)
|
||||
).to.equal(true)
|
||||
})
|
||||
|
||||
return it('should call the callback', function() {
|
||||
return expect(this.callback.called).to.equal(true)
|
||||
})
|
||||
})
|
||||
|
||||
describe('getLearnedWords', function() {
|
||||
beforeEach(function() {
|
||||
this.wordList = ['apples', 'bananas', 'pears']
|
||||
this.db.spellingPreferences.findOne = (conditions, callback) => {
|
||||
return callback(null, { learnedWords: this.wordList })
|
||||
}
|
||||
sinon.spy(this.db.spellingPreferences, 'findOne')
|
||||
return this.LearnedWordsManager.getLearnedWords(this.token, this.callback)
|
||||
})
|
||||
|
||||
it('should get the word list for the given user', function() {
|
||||
return expect(
|
||||
this.db.spellingPreferences.findOne.calledWith({ token: this.token })
|
||||
).to.equal(true)
|
||||
})
|
||||
|
||||
return it('should return the word list in the callback', function() {
|
||||
return expect(this.callback.calledWith(null, this.wordList)).to.equal(
|
||||
true
|
||||
)
|
||||
})
|
||||
})
|
||||
|
||||
describe('caching the result', function() {
|
||||
it('should use the cache first if it is primed', function(done) {
|
||||
this.wordList = ['apples', 'bananas', 'pears']
|
||||
this.cache.get.returns(this.wordList)
|
||||
this.db.spellingPreferences.findOne = sinon.stub()
|
||||
return this.LearnedWordsManager.getLearnedWords(
|
||||
this.token,
|
||||
(err, spellings) => {
|
||||
this.db.spellingPreferences.findOne.called.should.equal(false)
|
||||
assert.deepEqual(this.wordList, spellings)
|
||||
return done()
|
||||
}
|
||||
)
|
||||
})
|
||||
|
||||
it('should set the cache after hitting the db', function(done) {
|
||||
this.wordList = ['apples', 'bananas', 'pears']
|
||||
this.db.spellingPreferences.findOne = sinon
|
||||
.stub()
|
||||
.callsArgWith(1, null, { learnedWords: this.wordList })
|
||||
return this.LearnedWordsManager.getLearnedWords(
|
||||
this.token,
|
||||
(err, spellings) => {
|
||||
this.cache.set
|
||||
.calledWith(this.token, this.wordList)
|
||||
.should.equal(true)
|
||||
return done()
|
||||
}
|
||||
)
|
||||
})
|
||||
|
||||
return it('should break cache when update is called', function(done) {
|
||||
this.word = 'instanton'
|
||||
return this.LearnedWordsManager.learnWord(this.token, this.word, () => {
|
||||
this.cache.del.calledWith(this.token).should.equal(true)
|
||||
return done()
|
||||
})
|
||||
})
|
||||
})
|
||||
|
||||
return describe('deleteUsersLearnedWords', function() {
|
||||
beforeEach(function() {
|
||||
return (this.db.spellingPreferences.remove = sinon.stub().callsArgWith(1))
|
||||
})
|
||||
|
||||
return it('should get the word list for the given user', function(done) {
|
||||
return this.LearnedWordsManager.deleteUsersLearnedWords(
|
||||
this.token,
|
||||
() => {
|
||||
this.db.spellingPreferences.remove
|
||||
.calledWith({ token: this.token })
|
||||
.should.equal(true)
|
||||
return done()
|
||||
}
|
||||
)
|
||||
})
|
||||
})
|
||||
})
|
||||
271
services/spelling/test/unit/js/SpellingAPIManagerTests.js
Normal file
271
services/spelling/test/unit/js/SpellingAPIManagerTests.js
Normal file
|
|
@ -0,0 +1,271 @@
|
|||
/* eslint-disable
|
||||
handle-callback-err,
|
||||
no-undef
|
||||
*/
|
||||
// TODO: This file was created by bulk-decaffeinate.
|
||||
// Sanity-check the conversion and remove this comment.
|
||||
/*
|
||||
* decaffeinate suggestions:
|
||||
* DS102: Remove unnecessary code created because of implicit returns
|
||||
* Full docs: https://github.com/decaffeinate/decaffeinate/blob/master/docs/suggestions.md
|
||||
*/
|
||||
const sinon = require('sinon')
|
||||
const chai = require('chai')
|
||||
const { expect } = chai
|
||||
chai.should()
|
||||
const SandboxedModule = require('sandboxed-module')
|
||||
const modulePath = require('path').join(
|
||||
__dirname,
|
||||
'../../../app/js/SpellingAPIManager'
|
||||
)
|
||||
|
||||
describe('SpellingAPIManager', function() {
|
||||
beforeEach(function() {
|
||||
this.token = 'user-id-123'
|
||||
this.ASpell = {}
|
||||
this.learnedWords = ['lerned']
|
||||
this.LearnedWordsManager = {
|
||||
getLearnedWords: sinon.stub().callsArgWith(1, null, this.learnedWords),
|
||||
learnWord: sinon.stub().callsArg(2)
|
||||
}
|
||||
|
||||
return (this.SpellingAPIManager = SandboxedModule.require(modulePath, {
|
||||
requires: {
|
||||
'./ASpell': this.ASpell,
|
||||
'./LearnedWordsManager': this.LearnedWordsManager
|
||||
}
|
||||
}))
|
||||
})
|
||||
|
||||
describe('runRequest', function() {
|
||||
beforeEach(function() {
|
||||
this.nonLearnedWords = [
|
||||
'some',
|
||||
'words',
|
||||
'htat',
|
||||
'are',
|
||||
'speled',
|
||||
'rong',
|
||||
'lerned'
|
||||
]
|
||||
this.allWords = this.nonLearnedWords.concat(this.learnedWords)
|
||||
this.misspellings = [
|
||||
{ index: 2, suggestions: ['that'] },
|
||||
{ index: 4, suggestions: ['spelled'] },
|
||||
{ index: 5, suggestions: ['wrong', 'ring'] },
|
||||
{ index: 6, suggestions: ['learned'] }
|
||||
]
|
||||
this.misspellingsWithoutLearnedWords = this.misspellings.slice(0, 3)
|
||||
|
||||
this.ASpell.checkWords = (lang, word, callback) => {
|
||||
return callback(null, this.misspellings)
|
||||
}
|
||||
return sinon.spy(this.ASpell, 'checkWords')
|
||||
})
|
||||
|
||||
describe('with sensible JSON', function() {
|
||||
beforeEach(function(done) {
|
||||
return this.SpellingAPIManager.runRequest(
|
||||
this.token,
|
||||
{ words: this.allWords },
|
||||
(error, result) => {
|
||||
this.result = result
|
||||
return done()
|
||||
}
|
||||
)
|
||||
})
|
||||
|
||||
return it('should return the words that are spelled incorrectly and not learned', function() {
|
||||
return expect(this.result.misspellings).to.deep.equal(
|
||||
this.misspellingsWithoutLearnedWords
|
||||
)
|
||||
})
|
||||
})
|
||||
|
||||
describe('with a missing words array', function() {
|
||||
beforeEach(function(done) {
|
||||
return this.SpellingAPIManager.runRequest(
|
||||
this.token,
|
||||
{},
|
||||
(error, result) => {
|
||||
this.error = error
|
||||
this.result = result
|
||||
return done()
|
||||
}
|
||||
)
|
||||
})
|
||||
|
||||
return it('should return an error', function() {
|
||||
expect(this.error).to.exist
|
||||
expect(this.error).to.be.instanceof(Error)
|
||||
return expect(this.error.message).to.equal('malformed JSON')
|
||||
})
|
||||
})
|
||||
|
||||
describe('with a missing token', function() {
|
||||
beforeEach(function(done) {
|
||||
return this.SpellingAPIManager.runRequest(
|
||||
null,
|
||||
{ words: this.allWords },
|
||||
(error, result) => {
|
||||
this.error = error
|
||||
this.result = result
|
||||
return done()
|
||||
}
|
||||
)
|
||||
})
|
||||
|
||||
return it('should spell check without using any learned words', function() {
|
||||
return this.LearnedWordsManager.getLearnedWords.called.should.equal(
|
||||
false
|
||||
)
|
||||
})
|
||||
})
|
||||
|
||||
describe('without a language', function() {
|
||||
beforeEach(function(done) {
|
||||
return this.SpellingAPIManager.runRequest(
|
||||
this.token,
|
||||
{ words: this.allWords },
|
||||
(error, result) => {
|
||||
this.result = result
|
||||
return done()
|
||||
}
|
||||
)
|
||||
})
|
||||
|
||||
return it('should use en as the default', function() {
|
||||
return this.ASpell.checkWords.calledWith('en').should.equal(true)
|
||||
})
|
||||
})
|
||||
|
||||
describe('with a language', function() {
|
||||
beforeEach(function(done) {
|
||||
return this.SpellingAPIManager.runRequest(
|
||||
this.token,
|
||||
{
|
||||
words: this.allWords,
|
||||
language: (this.language = 'fr')
|
||||
},
|
||||
(error, result) => {
|
||||
this.result = result
|
||||
return done()
|
||||
}
|
||||
)
|
||||
})
|
||||
|
||||
return it('should use the language', function() {
|
||||
return this.ASpell.checkWords
|
||||
.calledWith(this.language)
|
||||
.should.equal(true)
|
||||
})
|
||||
})
|
||||
|
||||
describe('with a very large collection of words', function() {
|
||||
beforeEach(function(done) {
|
||||
this.manyWords = __range__(1, 100000, true).map(i => 'word')
|
||||
return this.SpellingAPIManager.runRequest(
|
||||
this.token,
|
||||
{ words: this.manyWords },
|
||||
(error, result) => {
|
||||
this.result = result
|
||||
return done()
|
||||
}
|
||||
)
|
||||
})
|
||||
|
||||
return it('should truncate to 10,000 words', function() {
|
||||
return this.ASpell.checkWords
|
||||
.calledWith(sinon.match.any, this.manyWords.slice(0, 10000))
|
||||
.should.equal(true)
|
||||
})
|
||||
})
|
||||
|
||||
return describe('with words from the whitelist', function() {
|
||||
beforeEach(function(done) {
|
||||
this.whitelistWord = this.SpellingAPIManager.whitelist[0]
|
||||
this.words = ['One', 'Two', this.whitelistWord]
|
||||
return this.SpellingAPIManager.runRequest(
|
||||
this.token,
|
||||
{ words: this.words },
|
||||
(error, result) => {
|
||||
this.result = result
|
||||
return done()
|
||||
}
|
||||
)
|
||||
})
|
||||
|
||||
return it('should ignore the white-listed word', function() {
|
||||
return expect(this.result.misspellings.length).to.equal(
|
||||
this.misspellings.length - 1
|
||||
)
|
||||
})
|
||||
})
|
||||
})
|
||||
|
||||
return describe('learnWord', function() {
|
||||
describe('without a token', function() {
|
||||
beforeEach(function(done) {
|
||||
return this.SpellingAPIManager.learnWord(
|
||||
null,
|
||||
{ word: 'banana' },
|
||||
error => {
|
||||
this.error = error
|
||||
return done()
|
||||
}
|
||||
)
|
||||
})
|
||||
|
||||
return it('should return an error', function() {
|
||||
expect(this.error).to.exist
|
||||
expect(this.error).to.be.instanceof(Error)
|
||||
return expect(this.error.message).to.equal('no token provided')
|
||||
})
|
||||
})
|
||||
|
||||
describe('without a word', function() {
|
||||
beforeEach(function(done) {
|
||||
return this.SpellingAPIManager.learnWord(this.token, {}, error => {
|
||||
this.error = error
|
||||
return done()
|
||||
})
|
||||
})
|
||||
|
||||
return it('should return an error', function() {
|
||||
expect(this.error).to.exist
|
||||
expect(this.error).to.be.instanceof(Error)
|
||||
return expect(this.error.message).to.equal('malformed JSON')
|
||||
})
|
||||
})
|
||||
|
||||
return describe('with a word and a token', function() {
|
||||
beforeEach(function(done) {
|
||||
this.word = 'banana'
|
||||
return this.SpellingAPIManager.learnWord(
|
||||
this.token,
|
||||
{ word: this.word },
|
||||
error => {
|
||||
this.error = error
|
||||
return done()
|
||||
}
|
||||
)
|
||||
})
|
||||
|
||||
return it('should call LearnedWordsManager.learnWord', function() {
|
||||
return this.LearnedWordsManager.learnWord
|
||||
.calledWith(this.token, this.word)
|
||||
.should.equal(true)
|
||||
})
|
||||
})
|
||||
})
|
||||
})
|
||||
|
||||
function __range__(left, right, inclusive) {
|
||||
let range = []
|
||||
let ascending = left < right
|
||||
let end = !inclusive ? right : ascending ? right + 1 : right - 1
|
||||
for (let i = left; ascending ? i < end : i > end; ascending ? i++ : i--) {
|
||||
range.push(i)
|
||||
}
|
||||
return range
|
||||
}
|
||||
Loading…
Reference in a new issue