github/overleaf
1
0
Fork 0
mirror of https://github.com/overleaf/overleaf.git synced 2024-11-21 20:47:08 -05:00
Code Issues Projects Releases Packages Wiki Activity

Initial decaffeination (#24)

Browse source
This commit is contained in:
Miguel Serrano 2019-07-03 13:41:01 +01:00 committed by GitHub
parent 1e74d545c4
commit 0eba057cef
43 changed files with 1840 additions and 940 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

2
services/spelling/.dockerignore
View file

@ -5,5 +5,3 @@ gitrev
.npm .npm
.nvmrc .nvmrc
nodemon.json nodemon.json
app.js
**/js/*

36
services/spelling/.eslintrc Normal file
View file

@ -0,0 +1,36 @@
// this file was auto-generated, do not edit it directly.
// instead run bin/update_build_scripts from
// https://github.com/sharelatex/sharelatex-dev-environment
// Version: 1.1.21
{
"extends": [
"standard",
"prettier",
"prettier/standard",
],
"plugins": [
"mocha",
"chai-expect",
"chai-friendly"
],
"env": {
"node": true
},
"rules": {
// Add some mocha specific rules
"mocha/handle-done-callback": "error",
"mocha/no-exclusive-tests": "error",
"mocha/no-global-tests": "error",
"mocha/no-identical-title": "error",
"mocha/no-nested-tests": "error",
"mocha/no-pending-tests": "error",
"mocha/no-skipped-tests": "error",
// Add some chai specific rules
"chai-expect/missing-assertion": "error",
"chai-expect/terminating-properties": "error",
// Swap the no-unused-expressions rule with a more chai-friendly one
"no-unused-expressions": 0,
"chai-friendly/no-unused-expressions": "error"
}
}

4
services/spelling/.gitignore vendored
View file

@ -1,9 +1,5 @@
**.swp **.swp
**.swo **.swo
app/js/*
app.js
test/UnitTests/js/*
node_modules/* node_modules/*
test/unit/js/
cache/spell.cache cache/spell.cache
**/*.map **/*.map

8
services/spelling/.prettierrc Normal file
View file

@ -0,0 +1,8 @@
# This file was auto-generated, do not edit it directly.
# Instead run bin/update_build_scripts from
# https://github.com/sharelatex/sharelatex-dev-environment
# Version: 1.1.21
{
"semi": false,
"singleQuote": true
}

1
services/spelling/Dockerfile
View file

@ -10,7 +10,6 @@ RUN npm install --quiet
COPY . /app COPY . /app
RUN npm run compile:all
FROM node:6.16.0 FROM node:6.16.0

8
services/spelling/Jenkinsfile vendored
View file

@ -36,6 +36,14 @@ pipeline {
} }
} }
// should be enabled once Node version is updated to >=8
// stage('Linting') {
// steps {
// sh 'DOCKER_COMPOSE_FLAGS="-f docker-compose.ci.yml" make format'
// sh 'DOCKER_COMPOSE_FLAGS="-f docker-compose.ci.yml" make lint'
// }
// }
stage('Unit Tests') { stage('Unit Tests') {
steps { steps {
sh 'DOCKER_COMPOSE_FLAGS="-f docker-compose.ci.yml" make test_unit' sh 'DOCKER_COMPOSE_FLAGS="-f docker-compose.ci.yml" make test_unit'

15
services/spelling/Makefile
View file

@ -16,12 +16,17 @@ DOCKER_COMPOSE := BUILD_NUMBER=$(BUILD_NUMBER) \
clean: clean:
docker rmi ci/$(PROJECT_NAME):$(BRANCH_NAME)-$(BUILD_NUMBER) docker rmi ci/$(PROJECT_NAME):$(BRANCH_NAME)-$(BUILD_NUMBER)
docker rmi gcr.io/overleaf-ops/$(PROJECT_NAME):$(BRANCH_NAME)-$(BUILD_NUMBER) docker rmi gcr.io/overleaf-ops/$(PROJECT_NAME):$(BRANCH_NAME)-$(BUILD_NUMBER)
rm -f app.js
rm -rf app/js
rm -rf test/unit/js
rm -rf test/acceptance/js
test: test_unit test_acceptance format:
$(DOCKER_COMPOSE) run --rm test_unit npm run format
format_fix:
$(DOCKER_COMPOSE) run --rm test_unit npm run format:fix
lint:
$(DOCKER_COMPOSE) run --rm test_unit npm run lint
test: format lint test_unit test_acceptance
test_unit: test_unit:
@[ ! -d test/unit ] && echo "spelling has no unit tests" || $(DOCKER_COMPOSE) run --rm test_unit @[ ! -d test/unit ] && echo "spelling has no unit tests" || $(DOCKER_COMPOSE) run --rm test_unit

47
services/spelling/app.coffee
View file

@ -1,47 +0,0 @@
metrics = require("metrics-sharelatex")
metrics.initialize("spelling")
Settings = require 'settings-sharelatex'
logger = require 'logger-sharelatex'
logger.initialize("spelling")
if Settings.sentry?.dsn?
logger.initializeErrorReporting(Settings.sentry.dsn)
metrics.memory.monitor(logger)
SpellingAPIController = require './app/js/SpellingAPIController'
express = require('express')
Path = require("path")
server = express()
metrics.injectMetricsRoute(server)
bodyParser = require('body-parser')
HealthCheckController = require("./app/js/HealthCheckController")
server.use bodyParser.json(limit: "2mb")
server.use metrics.http.monitor(logger)
server.del "/user/:user_id", SpellingAPIController.deleteDic
server.get "/user/:user_id", SpellingAPIController.getDic
server.post "/user/:user_id/check", SpellingAPIController.check
server.post "/user/:user_id/learn", SpellingAPIController.learn
server.get "/status", (req, res)->
res.send(status:'spelling api is up')
server.get "/health_check", HealthCheckController.healthCheck
profiler = require "v8-profiler"
server.get "/profile", (req, res) ->
time = parseInt(req.query.time || "1000")
profiler.startProfiling("test")
setTimeout () ->
profile = profiler.stopProfiling("test")
res.json(profile)
, time
host = Settings.internal?.spelling?.host || "localhost"
port = Settings.internal?.spelling?.port || 3005
server.listen port, host, (error) ->
throw error if error?
logger.info "spelling starting up, listening on #{host}:#{port}"

68
services/spelling/app.js Normal file
View file

@ -0,0 +1,68 @@
/*
* decaffeinate suggestions:
* DS102: Remove unnecessary code created because of implicit returns
* DS103: Rewrite code to no longer use __guard__
* DS207: Consider shorter variations of null checks
* Full docs: https://github.com/decaffeinate/decaffeinate/blob/master/docs/suggestions.md
*/
const metrics = require('metrics-sharelatex')
metrics.initialize('spelling')
const Settings = require('settings-sharelatex')
const logger = require('logger-sharelatex')
logger.initialize('spelling')
if ((Settings.sentry != null ? Settings.sentry.dsn : undefined) != null) {
logger.initializeErrorReporting(Settings.sentry.dsn)
}
metrics.memory.monitor(logger)
const SpellingAPIController = require('./app/js/SpellingAPIController')
const express = require('express')
const server = express()
metrics.injectMetricsRoute(server)
const bodyParser = require('body-parser')
const HealthCheckController = require('./app/js/HealthCheckController')
server.use(bodyParser.json({ limit: '2mb' }))
server.use(metrics.http.monitor(logger))
server.del('/user/:user_id', SpellingAPIController.deleteDic)
server.get('/user/:user_id', SpellingAPIController.getDic)
server.post('/user/:user_id/check', SpellingAPIController.check)
server.post('/user/:user_id/learn', SpellingAPIController.learn)
server.get('/status', (req, res) => res.send({ status: 'spelling api is up' }))
server.get('/health_check', HealthCheckController.healthCheck)
const profiler = require('v8-profiler')
server.get('/profile', function(req, res) {
const time = parseInt(req.query.time || '1000')
profiler.startProfiling('test')
return setTimeout(function() {
const profile = profiler.stopProfiling('test')
return res.json(profile)
}, time)
})
const host =
__guard__(
Settings.internal != null ? Settings.internal.spelling : undefined,
x => x.host
) || 'localhost'
const port =
__guard__(
Settings.internal != null ? Settings.internal.spelling : undefined,
x1 => x1.port
) || 3005
server.listen(port, host, function(error) {
if (error != null) {
throw error
}
return logger.info(`spelling starting up, listening on ${host}:${port}`)
})
function __guard__(value, transform) {
return typeof value !== 'undefined' && value !== null
? transform(value)
: undefined
}

119
services/spelling/app/coffee/ASpell.coffee
View file

@ -1,119 +0,0 @@
async = require "async"
ASpellWorkerPool = require "./ASpellWorkerPool"
LRU = require "lru-cache"
logger = require 'logger-sharelatex'
fs = require 'fs'
settings = require("settings-sharelatex")
Path = require("path")
OneMinute = 60 * 1000
opts = {max:10000, maxAge: OneMinute * 60 * 10}
cache = LRU(opts)
cacheFsPath = Path.resolve(settings.cacheDir, "spell.cache")
cacheFsPathTmp = cacheFsPath + ".tmp"
# load any existing cache
try
oldCache = fs.readFileSync cacheFsPath
cache.load JSON.parse(oldCache)
catch err
logger.log err:err, cacheFsPath:cacheFsPath, "could not load the cache file"
# write the cache every 30 minutes
setInterval () ->
dump = JSON.stringify cache.dump()
fs.writeFile cacheFsPathTmp, dump, (err) ->
if err?
logger.log {err}, "error writing cache file"
fs.unlink cacheFsPathTmp
else
fs.rename cacheFsPathTmp, cacheFsPath
logger.log {len: dump.length, cacheFsPath:cacheFsPath}, "wrote cache file"
, 30 * OneMinute
class ASpellRunner
checkWords: (language, words, callback = (error, result) ->) ->
@runAspellOnWords language, words, (error, output) =>
return callback(error) if error?
#output = @removeAspellHeader(output)
suggestions = @getSuggestions(language, output)
results = []
hits = 0
addToCache = {}
for word, i in words
key = language + ':' + word
cached = cache.get(key)
if cached?
hits++
if cached == true
# valid word, no need to do anything
continue
else
results.push index: i, suggestions: cached
else
if suggestions[key]?
addToCache[key] = suggestions[key]
results.push index: i, suggestions: suggestions[key]
else
# a valid word, but uncached
addToCache[key] = true
# update the cache after processing all words, to avoid cache
# changing while we use it
for k, v of addToCache
cache.set(k, v)
logger.info hits: hits, total: words.length, hitrate: (hits/words.length).toFixed(2), "cache hit rate"
callback null, results
getSuggestions: (language, output) ->
lines = output.split("\n")
suggestions = {}
for line in lines
if line[0] == "&" # Suggestions found
parts = line.split(" ")
if parts.length > 1
word = parts[1]
suggestionsString = line.slice(line.indexOf(":") + 2)
suggestions[language + ":" + word] = suggestionsString.split(", ")
else if line[0] == "#" # No suggestions
parts = line.split(" ")
if parts.length > 1
word = parts[1]
suggestions[language + ":" + word] = []
return suggestions
#removeAspellHeader: (output) -> output.slice(1)
runAspellOnWords: (language, words, callback = (error, output) ->) ->
# send words to aspell, get back string output for those words
# find a free pipe for the language (or start one)
# send the words down the pipe
# send an END marker that will generate a "*" line in the output
# when the output pipe receives the "*" return the data sofar and reset the pipe to be available
#
# @open(language)
# @captureOutput(callback)
# @setTerseMode()
# start = new Date()
newWord = {}
for word in words
newWord[word] = true if !newWord[word] && !cache.has(language + ':' + word)
words = Object.keys(newWord)
if words.length
WorkerPool.check(language, words, ASpell.ASPELL_TIMEOUT, callback)
else
callback null, ""
module.exports = ASpell =
# The description of how to call aspell from another program can be found here:
# http://aspell.net/man-html/Through-A-Pipe.html
checkWords: (language, words, callback = (error, result) ->) ->
runner = new ASpellRunner()
runner.checkWords language, words, callback
ASPELL_TIMEOUT : 4000
WorkerPool = new ASpellWorkerPool()

117
services/spelling/app/coffee/ASpellWorker.coffee
View file

@ -1,117 +0,0 @@
child_process = require("child_process")
logger = require 'logger-sharelatex'
metrics = require('metrics-sharelatex')
_ = require "underscore"
BATCH_SIZE = 100
class ASpellWorker
constructor: (language) ->
@language = language
@count = 0
@pipe = child_process.spawn("aspell", ["pipe", "-t", "--encoding=utf-8", "-d", language])
logger.info process: @pipe.pid, lang: @language, "starting new aspell worker"
metrics.inc "aspellWorker", 1, {status: "start", method: @language}
@pipe.on 'exit', () =>
@state = 'killed'
logger.info process: @pipe.pid, lang: @language, "aspell worker has exited"
metrics.inc "aspellWorker" , 1, {status: "exit", method: @language}
@pipe.on 'close', () =>
@state = 'closed' unless @state == 'killed'
if @callback?
logger.err process: @pipe.pid, lang: @language, "aspell worker closed output streams with uncalled callback"
@callback new Error("aspell worker closed output streams with uncalled callback"), []
@callback = null
@pipe.on 'error', (err) =>
@state = 'error' unless @state == 'killed'
logger.log process: @pipe.pid, error: err, stdout: output.slice(-1024), stderr: error.slice(-1024), lang: @language, "aspell worker error"
if @callback?
@callback err, []
@callback = null
@pipe.stdin.on 'error', (err) =>
@state = 'error' unless @state == 'killed'
logger.info process: @pipe.pid, error: err, stdout: output.slice(-1024), stderr: error.slice(-1024), lang: @language, "aspell worker error on stdin"
if @callback?
@callback err, []
@callback = null
output = ""
endMarker = new RegExp("^[a-z][a-z]", "m")
@pipe.stdout.on "data", (chunk) =>
output = output + chunk
# We receive the language code from Aspell as the end of data marker
if chunk.toString().match(endMarker)
if @callback?
@callback(null, output.slice())
@callback = null # only allow one callback in use
else
logger.err process: @pipe.pid, lang: @language, "end of data marker received when callback already used"
@state = 'ready'
output = ""
error = ""
error = ""
@pipe.stderr.on "data", (chunk) =>
error = error + chunk
@pipe.stdout.on "end", () =>
# process has ended
@state = "end"
isReady: () ->
return @state == 'ready'
check: (words, callback) ->
# we will now send data to aspell, and be ready again when we
# receive the end of data marker
@state = 'busy'
if @callback? # only allow one callback in use
logger.err process: @pipe.pid, lang: @language, "CALLBACK ALREADY IN USE"
return @callback new Error("Aspell callback already in use - SHOULD NOT HAPPEN")
@callback = _.once callback # extra defence against double callback
@setTerseMode()
@write(words)
@flush()
write: (words) ->
i = 0
while i < words.length
# batch up the words to check for efficiency
batch = words.slice(i, i + BATCH_SIZE)
@sendWords batch
i += BATCH_SIZE
flush: () ->
# get aspell to send an end of data marker "*" when ready
#@sendCommand("%") # take the aspell pipe out of terse mode so we can look for a '*'
#@sendCommand("^ENDOFSTREAMMARKER") # send our marker which will generate a '*'
#@sendCommand("!") # go back into terse mode
@sendCommand("$$l")
shutdown: (reason) ->
logger.info process: @pipe.pid, reason: reason, 'shutting down'
@state = "closing"
@pipe.stdin.end()
kill: (reason) ->
logger.info process: @pipe.pid, reason: reason, 'killing'
return if @state == 'killed'
@pipe.kill('SIGKILL')
setTerseMode: () ->
@sendCommand("!")
sendWord: (word) ->
@sendCommand("^" + word)
sendWords: (words) ->
# Aspell accepts multiple words to check on the same line
# ^word1 word2 word3 ...
# See aspell.info, writing programs to use Aspell Through A Pipe
@sendCommand("^" + words.join(" "))
@count++
sendCommand: (command) ->
@pipe.stdin.write(command + "\n")
module.exports = ASpellWorker

76
services/spelling/app/coffee/ASpellWorkerPool.coffee
View file

@ -1,76 +0,0 @@
ASpellWorker = require "./ASpellWorker"
_ = require "underscore"
logger = require 'logger-sharelatex'
metrics = require('metrics-sharelatex')
class ASpellWorkerPool
MAX_REQUESTS: 100*1024
MAX_WORKERS: 32
MAX_IDLE_TIME: 1000
MAX_REQUEST_TIME: 60*1000
constructor: (@options) ->
@PROCESS_POOL = []
create: (language) ->
if @PROCESS_POOL.length >= @MAX_WORKERS
logger.log maxworkers: @MAX_WORKERS, "maximum number of workers already running"
return null
worker = new ASpellWorker(language, @options)
worker.pipe.on 'exit', () =>
if worker.killTimer?
clearTimeout worker.killTimer
worker.killTimer = null
if worker.idleTimer?
clearTimeout worker.idleTimer
worker.idleTimer = null
logger.info process: worker.pipe.pid, lang: language, "removing aspell worker from pool"
@cleanup()
@PROCESS_POOL.push(worker)
metrics.gauge 'aspellWorkerPool-size', @PROCESS_POOL.length
return worker
cleanup: () ->
active = @PROCESS_POOL.filter (worker) ->
worker.state != 'killed'
@PROCESS_POOL = active
metrics.gauge 'aspellWorkerPool-size', @PROCESS_POOL.length
check: (language, words, timeout, callback) ->
# look for an existing process in the pool
availableWorker = _.find @PROCESS_POOL, (cached) ->
cached.language == language && cached.isReady()
if not availableWorker?
worker = @create(language)
else
worker = availableWorker
if not worker?
# return error if too many workers
callback(new Error("no worker available"))
return
if worker.idleTimer?
clearTimeout worker.idleTimer
worker.idleTimer = null
worker.killTimer = setTimeout () ->
worker.kill("spell check timed out")
, timeout || @MAX_REQUEST_TIME
worker.check words, (err, output) =>
if worker.killTimer?
clearTimeout worker.killTimer
worker.killTimer = null
callback(err, output)
return if err? # process has shut down
if worker.count > @MAX_REQUESTS
worker.shutdown("reached limit of " + @MAX_REQUESTS + " requests")
else
# queue a shutdown if worker is idle
worker.idleTimer = setTimeout () ->
worker.shutdown("idle worker")
worker.idleTimer = null
, @MAX_IDLE_TIME
module.exports = ASpellWorkerPool

4
services/spelling/app/coffee/DB.coffee
View file

@ -1,4 +0,0 @@
MongoJS = require "mongojs"
Settings = require "settings-sharelatex"
module.exports = MongoJS(Settings.mongo.url, ["spellingPreferences"])

23
services/spelling/app/coffee/HealthCheckController.coffee
View file

@ -1,23 +0,0 @@
request = require("request")
logger = require 'logger-sharelatex'
settings = require 'settings-sharelatex'
module.exports =
healthCheck: (req, res)->
opts =
url: "http://localhost:3005/user/#{settings.healthCheckUserId}/check"
json:
words:["helllo"]
language: "en"
timeout: 1000 * 20
request.post opts, (err, response, body)->
if err?
return res.sendStatus 500
numberOfSuggestions = body?.misspellings?[0]?.suggestions?.length
if numberOfSuggestions > 10
logger.log "health check passed"
res.sendStatus 200
else
logger.err body:body, numberOfSuggestions:numberOfSuggestions, "health check failed"
res.sendStatus 500

40
services/spelling/app/coffee/LearnedWordsManager.coffee
View file

@ -1,40 +0,0 @@
db = require("./DB")
mongoCache = require("./MongoCache")
logger = require 'logger-sharelatex'
metrics = require('metrics-sharelatex')
module.exports = LearnedWordsManager =
learnWord: (user_token, word, callback = (error)->) ->
mongoCache.del(user_token)
db.spellingPreferences.update {
token: user_token
}, {
$push: learnedWords: word
}, {
upsert: true
}, callback
getLearnedWords: (user_token, callback = (error, words)->) ->
mongoCachedWords = mongoCache.get(user_token)
if mongoCachedWords?
metrics.inc "mongoCache", 0.1, {status: "hit"}
return callback(null, mongoCachedWords)
metrics.inc "mongoCache", 0.1, {status: "miss"}
logger.info user_token:user_token, "mongoCache miss"
db.spellingPreferences.findOne token: user_token, (error, preferences) ->
return callback error if error?
words = preferences?.learnedWords || []
mongoCache.set(user_token, words)
callback null, words
deleteUsersLearnedWords: (user_token, callback =(error)->)->
db.spellingPreferences.remove token: user_token, callback
[
'learnWord',
'getLearnedWords'
].map (method) ->
metrics.timeAsyncMethod(LearnedWordsManager, method, 'mongo.LearnedWordsManager', logger)

8
services/spelling/app/coffee/MongoCache.coffee
View file

@ -1,8 +0,0 @@
LRU = require("lru-cache")
cacheOpts =
max: 15000
maxAge: 1000 * 60 * 60 * 10
cache = LRU(cacheOpts)
module.exports = cache

35
services/spelling/app/coffee/SpellingAPIController.coffee
View file

@ -1,35 +0,0 @@
SpellingAPIManager = require './SpellingAPIManager'
logger = require 'logger-sharelatex'
metrics = require('metrics-sharelatex')
module.exports = SpellingAPIController =
check: (req, res, next) ->
metrics.inc "spelling-check", 0.1
logger.info token: req?.params?.user_id, word_count: req?.body?.words?.length, "running check"
SpellingAPIManager.runRequest req.params.user_id, req.body, (error, result) ->
if error?
logger.err err:error, user_id:req?.params?.user_id, word_count: req?.body?.words?.length, "error processing spelling request"
return res.sendStatus(500)
res.send(result)
learn: (req, res, next) ->
metrics.inc "spelling-learn", 0.1
logger.info token: req?.params?.user_id, word: req?.body?.word, "learning word"
SpellingAPIManager.learnWord req.params.user_id, req.body, (error, result) ->
return next(error) if error?
res.sendStatus(200)
next()
deleteDic: (req, res, next)->
logger.log token: req?.params?.user_id, word: req?.body?.word, "deleting user dictionary"
SpellingAPIManager.deleteDic req.params.user_id, (error) ->
return next(error) if error?
res.sendStatus(204)
getDic: (req, res, next)->
logger.info token: req?.params?.user_id, "getting user dictionary"
SpellingAPIManager.getDic req.params.user_id, (error, words)->
return next(error) if error?
res.send(words)

54
services/spelling/app/coffee/SpellingAPIManager.coffee
View file

@ -1,54 +0,0 @@
ASpell = require './ASpell'
LearnedWordsManager = require './LearnedWordsManager'
async = require 'async'
module.exports = SpellingAPIManager =
whitelist: [
'ShareLaTeX',
'sharelatex',
'LaTeX',
'http',
'https',
'www'
]
runRequest: (token, request, callback = (error, result) ->) ->
if !request.words?
return callback(new Error("malformed JSON"))
lang = request.language || "en"
check = (words, callback) ->
ASpell.checkWords lang, words, (error, misspellings) ->
callback error, misspellings: misspellings
wordsToCheck = request.words || []
if token?
LearnedWordsManager.getLearnedWords token, (error, learnedWords) ->
return callback(error) if error?
words = (wordsToCheck).slice(0,10000)
check words, (error, result) ->
return callback error if error?
result.misspellings = result.misspellings.filter (m) ->
word = words[m.index]
learnedWords.indexOf(word) == -1 and SpellingAPIManager.whitelist.indexOf(word) == -1
callback error, result
else
check(wordsToCheck, callback)
learnWord: (token, request, callback = (error) ->) ->
if !request.word?
return callback(new Error("malformed JSON"))
if !token?
return callback(new Error("no token provided"))
LearnedWordsManager.learnWord token, request.word, callback
deleteDic: (token, callback)->
LearnedWordsManager.deleteUsersLearnedWords token, callback
getDic: (token, callback)->
LearnedWordsManager.getLearnedWords token, callback

175
services/spelling/app/js/ASpell.js Normal file
View file

@ -0,0 +1,175 @@
// TODO: This file was created by bulk-decaffeinate.
// Sanity-check the conversion and remove this comment.
/*
* decaffeinate suggestions:
* DS101: Remove unnecessary use of Array.from
* DS102: Remove unnecessary code created because of implicit returns
* DS207: Consider shorter variations of null checks
* Full docs: https://github.com/decaffeinate/decaffeinate/blob/master/docs/suggestions.md
*/
let ASpell
const ASpellWorkerPool = require('./ASpellWorkerPool')
const LRU = require('lru-cache')
const logger = require('logger-sharelatex')
const fs = require('fs')
const settings = require('settings-sharelatex')
const Path = require('path')
const OneMinute = 60 * 1000
const opts = { max: 10000, maxAge: OneMinute * 60 * 10 }
const cache = LRU(opts)
const cacheFsPath = Path.resolve(settings.cacheDir, 'spell.cache')
const cacheFsPathTmp = cacheFsPath + '.tmp'
// load any existing cache
try {
const oldCache = fs.readFileSync(cacheFsPath)
cache.load(JSON.parse(oldCache))
} catch (error) {
const err = error
logger.log({ err, cacheFsPath }, 'could not load the cache file')
}
// write the cache every 30 minutes
setInterval(function() {
const dump = JSON.stringify(cache.dump())
return fs.writeFile(cacheFsPathTmp, dump, function(err) {
if (err != null) {
logger.log({ err }, 'error writing cache file')
return fs.unlink(cacheFsPathTmp)
} else {
fs.rename(cacheFsPathTmp, cacheFsPath)
return logger.log({ len: dump.length, cacheFsPath }, 'wrote cache file')
}
})
}, 30 * OneMinute)
class ASpellRunner {
checkWords(language, words, callback) {
if (callback == null) {
callback = () => {}
}
return this.runAspellOnWords(language, words, (error, output) => {
if (error != null) {
return callback(error)
}
// output = @removeAspellHeader(output)
const suggestions = this.getSuggestions(language, output)
const results = []
let hits = 0
const addToCache = {}
for (let i = 0; i < words.length; i++) {
const word = words[i]
const key = language + ':' + word
const cached = cache.get(key)
if (cached != null) {
hits++
if (cached === true) {
// valid word, no need to do anything
continue
} else {
results.push({ index: i, suggestions: cached })
}
} else {
if (suggestions[key] != null) {
addToCache[key] = suggestions[key]
results.push({ index: i, suggestions: suggestions[key] })
} else {
// a valid word, but uncached
addToCache[key] = true
}
}
}
// update the cache after processing all words, to avoid cache
// changing while we use it
for (let k in addToCache) {
const v = addToCache[k]
cache.set(k, v)
}
logger.info(
{
hits,
total: words.length,
hitrate: (hits / words.length).toFixed(2)
},
'cache hit rate'
)
return callback(null, results)
})
}
getSuggestions(language, output) {
const lines = output.split('\n')
const suggestions = {}
for (let line of Array.from(lines)) {
var parts, word
if (line[0] === '&') {
// Suggestions found
parts = line.split(' ')
if (parts.length > 1) {
word = parts[1]
const suggestionsString = line.slice(line.indexOf(':') + 2)
suggestions[language + ':' + word] = suggestionsString.split(', ')
}
} else if (line[0] === '#') {
// No suggestions
parts = line.split(' ')
if (parts.length > 1) {
word = parts[1]
suggestions[language + ':' + word] = []
}
}
}
return suggestions
}
// removeAspellHeader: (output) -> output.slice(1)
runAspellOnWords(language, words, callback) {
// send words to aspell, get back string output for those words
// find a free pipe for the language (or start one)
// send the words down the pipe
// send an END marker that will generate a "*" line in the output
// when the output pipe receives the "*" return the data sofar and reset the pipe to be available
//
// @open(language)
// @captureOutput(callback)
// @setTerseMode()
// start = new Date()
if (callback == null) {
callback = () => {}
}
const newWord = {}
for (let word of Array.from(words)) {
if (!newWord[word] && !cache.has(language + ':' + word)) {
newWord[word] = true
}
}
words = Object.keys(newWord)
if (words.length) {
return WorkerPool.check(language, words, ASpell.ASPELL_TIMEOUT, callback)
} else {
return callback(null, '')
}
}
}
module.exports = ASpell = {
// The description of how to call aspell from another program can be found here:
// http://aspell.net/man-html/Through-A-Pipe.html
checkWords(language, words, callback) {
if (callback == null) {
callback = () => {}
}
const runner = new ASpellRunner()
return runner.checkWords(language, words, callback)
},
ASPELL_TIMEOUT: 4000
}
var WorkerPool = new ASpellWorkerPool()

213
services/spelling/app/js/ASpellWorker.js Normal file
View file

@ -0,0 +1,213 @@
// TODO: This file was created by bulk-decaffeinate.
// Sanity-check the conversion and remove this comment.
/*
* decaffeinate suggestions:
* DS102: Remove unnecessary code created because of implicit returns
* DS205: Consider reworking code to avoid use of IIFEs
* DS207: Consider shorter variations of null checks
* Full docs: https://github.com/decaffeinate/decaffeinate/blob/master/docs/suggestions.md
*/
const childProcess = require('child_process')
const logger = require('logger-sharelatex')
const metrics = require('metrics-sharelatex')
const _ = require('underscore')
const BATCH_SIZE = 100
class ASpellWorker {
constructor(language) {
this.language = language
this.count = 0
this.pipe = childProcess.spawn('aspell', [
'pipe',
'-t',
'--encoding=utf-8',
'-d',
language
])
logger.info(
{ process: this.pipe.pid, lang: this.language },
'starting new aspell worker'
)
metrics.inc('aspellWorker', 1, { status: 'start', method: this.language })
this.pipe.on('exit', () => {
this.state = 'killed'
logger.info(
{ process: this.pipe.pid, lang: this.language },
'aspell worker has exited'
)
return metrics.inc('aspellWorker', 1, {
status: 'exit',
method: this.language
})
})
this.pipe.on('close', () => {
if (this.state !== 'killed') {
this.state = 'closed'
}
if (this.callback != null) {
logger.err(
{ process: this.pipe.pid, lang: this.language },
'aspell worker closed output streams with uncalled callback'
)
this.callback(
new Error(
'aspell worker closed output streams with uncalled callback'
),
[]
)
return (this.callback = null)
}
})
this.pipe.on('error', err => {
if (this.state !== 'killed') {
this.state = 'error'
}
logger.log(
{
process: this.pipe.pid,
error: err,
stdout: output.slice(-1024),
stderr: error.slice(-1024),
lang: this.language
},
'aspell worker error'
)
if (this.callback != null) {
this.callback(err, [])
return (this.callback = null)
}
})
this.pipe.stdin.on('error', err => {
if (this.state !== 'killed') {
this.state = 'error'
}
logger.info(
{
process: this.pipe.pid,
error: err,
stdout: output.slice(-1024),
stderr: error.slice(-1024),
lang: this.language
},
'aspell worker error on stdin'
)
if (this.callback != null) {
this.callback(err, [])
return (this.callback = null)
}
})
var output = ''
const endMarker = new RegExp('^[a-z][a-z]', 'm')
this.pipe.stdout.on('data', chunk => {
output = output + chunk
// We receive the language code from Aspell as the end of data marker
if (chunk.toString().match(endMarker)) {
if (this.callback != null) {
this.callback(null, output.slice())
this.callback = null // only allow one callback in use
} else {
logger.err(
{ process: this.pipe.pid, lang: this.language },
'end of data marker received when callback already used'
)
}
this.state = 'ready'
output = ''
}
})
var error = ''
this.pipe.stderr.on('data', chunk => {
return (error = error + chunk)
})
this.pipe.stdout.on('end', () => {
// process has ended
return (this.state = 'end')
})
}
isReady() {
return this.state === 'ready'
}
check(words, callback) {
// we will now send data to aspell, and be ready again when we
// receive the end of data marker
this.state = 'busy'
if (this.callback != null) {
// only allow one callback in use
logger.err(
{ process: this.pipe.pid, lang: this.language },
'CALLBACK ALREADY IN USE'
)
return this.callback(
new Error('Aspell callback already in use - SHOULD NOT HAPPEN')
)
}
this.callback = _.once(callback) // extra defence against double callback
this.setTerseMode()
this.write(words)
return this.flush()
}
write(words) {
let i = 0
return (() => {
const result = []
while (i < words.length) {
// batch up the words to check for efficiency
const batch = words.slice(i, i + BATCH_SIZE)
this.sendWords(batch)
result.push((i += BATCH_SIZE))
}
return result
})()
}
flush() {
// get aspell to send an end of data marker "*" when ready
// @sendCommand("%") # take the aspell pipe out of terse mode so we can look for a '*'
// @sendCommand("^ENDOFSTREAMMARKER") # send our marker which will generate a '*'
// @sendCommand("!") # go back into terse mode
return this.sendCommand('$$l')
}
shutdown(reason) {
logger.info({ process: this.pipe.pid, reason }, 'shutting down')
this.state = 'closing'
return this.pipe.stdin.end()
}
kill(reason) {
logger.info({ process: this.pipe.pid, reason }, 'killing')
if (this.state === 'killed') {
return
}
return this.pipe.kill('SIGKILL')
}
setTerseMode() {
return this.sendCommand('!')
}
sendWord(word) {
return this.sendCommand(`^${word}`)
}
sendWords(words) {
// Aspell accepts multiple words to check on the same line
// ^word1 word2 word3 ...
// See aspell.info, writing programs to use Aspell Through A Pipe
this.sendCommand(`^${words.join(' ')}`)
return this.count++
}
sendCommand(command) {
return this.pipe.stdin.write(command + '\n')
}
}
module.exports = ASpellWorker

115
services/spelling/app/js/ASpellWorkerPool.js Normal file
View file

@ -0,0 +1,115 @@
// TODO: This file was created by bulk-decaffeinate.
// Sanity-check the conversion and remove this comment.
/*
* decaffeinate suggestions:
* DS102: Remove unnecessary code created because of implicit returns
* DS206: Consider reworking classes to avoid initClass
* DS207: Consider shorter variations of null checks
* Full docs: https://github.com/decaffeinate/decaffeinate/blob/master/docs/suggestions.md
*/
const ASpellWorker = require('./ASpellWorker')
const _ = require('underscore')
const logger = require('logger-sharelatex')
const metrics = require('metrics-sharelatex')
class ASpellWorkerPool {
static initClass() {
this.prototype.MAX_REQUESTS = 100 * 1024
this.prototype.MAX_WORKERS = 32
this.prototype.MAX_IDLE_TIME = 1000
this.prototype.MAX_REQUEST_TIME = 60 * 1000
}
constructor(options) {
this.options = options
this.PROCESS_POOL = []
}
create(language) {
if (this.PROCESS_POOL.length >= this.MAX_WORKERS) {
logger.log(
{ maxworkers: this.MAX_WORKERS },
'maximum number of workers already running'
)
return null
}
const worker = new ASpellWorker(language, this.options)
worker.pipe.on('exit', () => {
if (worker.killTimer != null) {
clearTimeout(worker.killTimer)
worker.killTimer = null
}
if (worker.idleTimer != null) {
clearTimeout(worker.idleTimer)
worker.idleTimer = null
}
logger.info(
{ process: worker.pipe.pid, lang: language },
'removing aspell worker from pool'
)
return this.cleanup()
})
this.PROCESS_POOL.push(worker)
metrics.gauge('aspellWorkerPool-size', this.PROCESS_POOL.length)
return worker
}
cleanup() {
const active = this.PROCESS_POOL.filter(worker => worker.state !== 'killed')
this.PROCESS_POOL = active
return metrics.gauge('aspellWorkerPool-size', this.PROCESS_POOL.length)
}
check(language, words, timeout, callback) {
// look for an existing process in the pool
let worker
const availableWorker = _.find(
this.PROCESS_POOL,
cached => cached.language === language && cached.isReady()
)
if (availableWorker == null) {
worker = this.create(language)
} else {
worker = availableWorker
}
if (worker == null) {
// return error if too many workers
callback(new Error('no worker available'))
return
}
if (worker.idleTimer != null) {
clearTimeout(worker.idleTimer)
worker.idleTimer = null
}
worker.killTimer = setTimeout(
() => worker.kill('spell check timed out'),
timeout || this.MAX_REQUEST_TIME
)
return worker.check(words, (err, output) => {
if (worker.killTimer != null) {
clearTimeout(worker.killTimer)
worker.killTimer = null
}
callback(err, output)
if (err != null) {
return
} // process has shut down
if (worker.count > this.MAX_REQUESTS) {
return worker.shutdown(`reached limit of ${this.MAX_REQUESTS} requests`)
} else {
// queue a shutdown if worker is idle
return (worker.idleTimer = setTimeout(function() {
worker.shutdown('idle worker')
return (worker.idleTimer = null)
}, this.MAX_IDLE_TIME))
}
})
}
}
ASpellWorkerPool.initClass()
module.exports = ASpellWorkerPool

5
services/spelling/app/js/DB.js Normal file
View file

@ -0,0 +1,5 @@
// TODO: This file was created by bulk-decaffeinate.
// Sanity-check the conversion and remove this comment.
const MongoJS = require('mongojs')
const Settings = require('settings-sharelatex')
module.exports = MongoJS(Settings.mongo.url, ['spellingPreferences'])

50
services/spelling/app/js/HealthCheckController.js Normal file
View file

@ -0,0 +1,50 @@
// TODO: This file was created by bulk-decaffeinate.
// Sanity-check the conversion and remove this comment.
/*
* decaffeinate suggestions:
* DS102: Remove unnecessary code created because of implicit returns
* DS103: Rewrite code to no longer use __guard__
* DS207: Consider shorter variations of null checks
* Full docs: https://github.com/decaffeinate/decaffeinate/blob/master/docs/suggestions.md
*/
const request = require('request')
const logger = require('logger-sharelatex')
const settings = require('settings-sharelatex')
module.exports = {
healthCheck(req, res) {
const opts = {
url: `http://localhost:3005/user/${settings.healthCheckUserId}/check`,
json: {
words: ['helllo'],
language: 'en'
},
timeout: 1000 * 20
}
return request.post(opts, function(err, response, body) {
if (err != null) {
return res.sendStatus(500)
}
const numberOfSuggestions = __guard__(
__guard__(
__guard__(body != null ? body.misspellings : undefined, x2 => x2[0]),
x1 => x1.suggestions
),
x => x.length
)
if (numberOfSuggestions > 10) {
logger.log('health check passed')
return res.sendStatus(200)
} else {
logger.err({ body, numberOfSuggestions }, 'health check failed')
return res.sendStatus(500)
}
})
}
}
function __guard__(value, transform) {
return typeof value !== 'undefined' && value !== null
? transform(value)
: undefined
}

76
services/spelling/app/js/LearnedWordsManager.js Normal file
View file

@ -0,0 +1,76 @@
// TODO: This file was created by bulk-decaffeinate.
// Sanity-check the conversion and remove this comment.
/*
* decaffeinate suggestions:
* DS102: Remove unnecessary code created because of implicit returns
* DS207: Consider shorter variations of null checks
* Full docs: https://github.com/decaffeinate/decaffeinate/blob/master/docs/suggestions.md
*/
let LearnedWordsManager
const db = require('./DB')
const mongoCache = require('./MongoCache')
const logger = require('logger-sharelatex')
const metrics = require('metrics-sharelatex')
module.exports = LearnedWordsManager = {
learnWord(userToken, word, callback) {
if (callback == null) {
callback = () => {}
}
mongoCache.del(userToken)
return db.spellingPreferences.update(
{
token: userToken
},
{
$push: { learnedWords: word }
},
{
upsert: true
},
callback
)
},
getLearnedWords(userToken, callback) {
if (callback == null) {
callback = () => {}
}
const mongoCachedWords = mongoCache.get(userToken)
if (mongoCachedWords != null) {
metrics.inc('mongoCache', 0.1, { status: 'hit' })
return callback(null, mongoCachedWords)
}
metrics.inc('mongoCache', 0.1, { status: 'miss' })
logger.info({ userToken }, 'mongoCache miss')
return db.spellingPreferences.findOne({ token: userToken }, function(
error,
preferences
) {
if (error != null) {
return callback(error)
}
const words =
(preferences != null ? preferences.learnedWords : undefined) || []
mongoCache.set(userToken, words)
return callback(null, words)
})
},
deleteUsersLearnedWords(userToken, callback) {
if (callback == null) {
callback = () => {}
}
return db.spellingPreferences.remove({ token: userToken }, callback)
}
}
;['learnWord', 'getLearnedWords'].map(method =>
metrics.timeAsyncMethod(
LearnedWordsManager,
method,
'mongo.LearnedWordsManager',
logger
)
)

11
services/spelling/app/js/MongoCache.js Normal file
View file

@ -0,0 +1,11 @@
// TODO: This file was created by bulk-decaffeinate.
// Sanity-check the conversion and remove this comment.
const LRU = require('lru-cache')
const cacheOpts = {
max: 15000,
maxAge: 1000 * 60 * 60 * 10
}
const cache = LRU(cacheOpts)
module.exports = cache

111
services/spelling/app/js/SpellingAPIController.js Normal file
View file

@ -0,0 +1,111 @@
// TODO: This file was created by bulk-decaffeinate.
// Sanity-check the conversion and remove this comment.
/*
* decaffeinate suggestions:
* DS102: Remove unnecessary code created because of implicit returns
* DS103: Rewrite code to no longer use __guard__
* DS207: Consider shorter variations of null checks
* Full docs: https://github.com/decaffeinate/decaffeinate/blob/master/docs/suggestions.md
*/
const SpellingAPIManager = require('./SpellingAPIManager')
const logger = require('logger-sharelatex')
const metrics = require('metrics-sharelatex')
module.exports = {
check(req, res, next) {
metrics.inc('spelling-check', 0.1)
logger.info(
{
token: __guard__(req != null ? req.params : undefined, x => x.user_id),
word_count: __guard__(
__guard__(req != null ? req.body : undefined, x2 => x2.words),
x1 => x1.length
)
},
'running check'
)
return SpellingAPIManager.runRequest(req.params.user_id, req.body, function(
error,
result
) {
if (error != null) {
logger.err(
{
err: error,
user_id: __guard__(
req != null ? req.params : undefined,
x3 => x3.user_id
),
word_count: __guard__(
__guard__(req != null ? req.body : undefined, x5 => x5.words),
x4 => x4.length
)
},
'error processing spelling request'
)
return res.sendStatus(500)
}
return res.send(result)
})
},
learn(req, res, next) {
metrics.inc('spelling-learn', 0.1)
logger.info(
{
token: __guard__(req != null ? req.params : undefined, x => x.user_id),
word: __guard__(req != null ? req.body : undefined, x1 => x1.word)
},
'learning word'
)
return SpellingAPIManager.learnWord(req.params.user_id, req.body, function(
error,
result
) {
if (error != null) {
return next(error)
}
res.sendStatus(200)
return next()
})
},
deleteDic(req, res, next) {
logger.log(
{
token: __guard__(req != null ? req.params : undefined, x => x.user_id),
word: __guard__(req != null ? req.body : undefined, x1 => x1.word)
},
'deleting user dictionary'
)
return SpellingAPIManager.deleteDic(req.params.user_id, function(error) {
if (error != null) {
return next(error)
}
return res.sendStatus(204)
})
},
getDic(req, res, next) {
logger.info(
{
token: __guard__(req != null ? req.params : undefined, x => x.user_id)
},
'getting user dictionary'
)
return SpellingAPIManager.getDic(req.params.user_id, function(
error,
words
) {
if (error != null) {
return next(error)
}
return res.send(words)
})
}
}
function __guard__(value, transform) {
return typeof value !== 'undefined' && value !== null
? transform(value)
: undefined
}

81
services/spelling/app/js/SpellingAPIManager.js Normal file
View file

@ -0,0 +1,81 @@
// TODO: This file was created by bulk-decaffeinate.
// Sanity-check the conversion and remove this comment.
/*
* decaffeinate suggestions:
* DS102: Remove unnecessary code created because of implicit returns
* DS207: Consider shorter variations of null checks
* Full docs: https://github.com/decaffeinate/decaffeinate/blob/master/docs/suggestions.md
*/
let SpellingAPIManager
const ASpell = require('./ASpell')
const LearnedWordsManager = require('./LearnedWordsManager')
module.exports = SpellingAPIManager = {
whitelist: ['ShareLaTeX', 'sharelatex', 'LaTeX', 'http', 'https', 'www'],
runRequest(token, request, callback) {
if (callback == null) {
callback = () => {}
}
if (request.words == null) {
return callback(new Error('malformed JSON'))
}
const lang = request.language || 'en'
const check = (words, callback) =>
ASpell.checkWords(lang, words, (error, misspellings) =>
callback(error, { misspellings })
)
const wordsToCheck = request.words || []
if (token != null) {
return LearnedWordsManager.getLearnedWords(token, function(
error,
learnedWords
) {
if (error != null) {
return callback(error)
}
const words = wordsToCheck.slice(0, 10000)
return check(words, function(error, result) {
if (error != null) {
return callback(error)
}
result.misspellings = result.misspellings.filter(function(m) {
const word = words[m.index]
return (
learnedWords.indexOf(word) === -1 &&
SpellingAPIManager.whitelist.indexOf(word) === -1
)
})
return callback(error, result)
})
})
} else {
return check(wordsToCheck, callback)
}
},
learnWord(token, request, callback) {
if (callback == null) {
callback = () => {}
}
if (request.word == null) {
return callback(new Error('malformed JSON'))
}
if (token == null) {
return callback(new Error('no token provided'))
}
return LearnedWordsManager.learnWord(token, request.word, callback)
},
deleteDic(token, callback) {
return LearnedWordsManager.deleteUsersLearnedWords(token, callback)
},
getDic(token, callback) {
return LearnedWordsManager.getLearnedWords(token, callback)
}
}

10
services/spelling/buildscript.txt
View file

@ -1,8 +1,8 @@
spelling spelling
--language=coffeescript
--node-version=6.16.0 --node-version=6.16.0
--acceptance-creds=None
--dependencies=mongo,redis
--docker-repos=gcr.io/overleaf-ops
--build-target=docker
--script-version=1.1.21 --script-version=1.1.21
--build-target=docker
--dependencies=mongo,redis
--language=es
--docker-repos=gcr.io/overleaf-ops
--acceptance-creds=None

18
services/spelling/config/settings.defaults.coffee
View file

@ -1,18 +0,0 @@
Path = require('path')
module.exports = Settings =
internal:
spelling:
port: 3005
host: process.env["LISTEN_ADDRESS"] or "localhost"
mongo:
url: process.env['MONGO_CONNECTION_STRING'] or "mongodb://#{process.env["MONGO_HOST"] or "localhost"}/sharelatex"
cacheDir: Path.resolve "cache"
healthCheckUserId: "53c64d2fd68c8d000010bb5f"
sentry:
dsn: process.env.SENTRY_DSN

24
services/spelling/config/settings.defaults.js Normal file
View file

@ -0,0 +1,24 @@
const Path = require('path')
module.exports = {
internal: {
spelling: {
port: 3005,
host: process.env['LISTEN_ADDRESS'] || 'localhost'
}
},
mongo: {
url:
process.env['MONGO_CONNECTION_STRING'] ||
`mongodb://${process.env['MONGO_HOST'] || 'localhost'}/sharelatex`
},
cacheDir: Path.resolve('cache'),
healthCheckUserId: '53c64d2fd68c8d000010bb5f',
sentry: {
dsn: process.env.SENTRY_DSN
}
}

79
services/spelling/decaffeinate.sh Normal file
View file

@ -0,0 +1,79 @@
#!/usr/bin/env bash
set -ex
# Check .eslintrc and .prettierc are present
npm install --save-dev eslint eslint-config-prettier eslint-config-standard \
eslint-plugin-chai-expect eslint-plugin-chai-friendly eslint-plugin-import \
eslint-plugin-mocha eslint-plugin-node eslint-plugin-prettier \
eslint-plugin-promise eslint-plugin-standard prettier-eslint-cli
git add .
git commit -m "Decaffeinate: add eslint and prettier rc files"
echo "------------------------"
echo "----------APP-----------"
echo "------------------------"
# bulk-decaffeinate will commit for you
npx bulk-decaffeinate convert --dir app/coffee
npx bulk-decaffeinate clean
git mv app/coffee app/js
git commit -m "Rename app/coffee dir to app/js"
npx prettier-eslint 'app/js/**/*.js' --write
git add .
git commit -m "Prettier: convert app/js decaffeinated files to Prettier format"
echo "-------------------------"
echo "--------UNIT TESTS-------"
echo "-------------------------"
npx bulk-decaffeinate convert --dir test/unit/coffee
npx bulk-decaffeinate clean
git mv test/unit/coffee test/unit/js
git commit -m "Rename test/unit/coffee to test/unit/js"
npx prettier-eslint 'test/unit/js/**/*.js' --write
git add .
git commit -m "Prettier: convert test/unit decaffeinated files to Prettier format"
echo "-------------------------"
echo "-------STRESS TESTS------"
echo "-------------------------"
npx bulk-decaffeinate convert --dir test/stress/coffee
npx bulk-decaffeinate clean
git mv test/stress/coffee test/stress/js
git commit -m "Rename test/stress/coffee to test/stress/js"
npx prettier-eslint 'test/stress/js/**/*.js' --write
git add .
git commit -m "Prettier: convert test/stress decaffeinated files to Prettier format"
echo "--------------------------"
echo "-----INDIVIDUAL FILES-----"
echo "--------------------------"
rm -f app.js config/settings.defaults.js
git mv app.coffee app.js
git mv config/settings.defaults.coffee config/settings.defaults.js
git commit -m "Rename individual coffee files to js files"
decaffeinate app.js
decaffeinate config/settings.defaults.js
git add .
git commit -m "Decaffeinate: convert individual files to js"
npx prettier-eslint 'app.js' 'config/settings.defaults.js' --write
git add .
git commit -m "Prettier: convert individual decaffeinated files to Prettier format"
echo "done."

7
services/spelling/nodemon.json
View file

@ -10,10 +10,9 @@
}, },
"watch": [ "watch": [
"app/coffee/", "app/js/",
"app.coffee", "app.js",
"config/" "config/"
], ],
"ext": "coffee" "ext": "js"
} }

25
services/spelling/package.json
View file

@ -8,16 +8,19 @@
}, },
"scripts": { "scripts": {
"compile:app": "([ -e app/coffee ] && coffee -m $COFFEE_OPTIONS -o app/js -c app/coffee || echo 'No CoffeeScript folder to compile') && ( [ -e app.coffee ] && coffee -m $COFFEE_OPTIONS -c app.coffee || echo 'No CoffeeScript app to compile')", "compile:app": "([ -e app/coffee ] && coffee -m $COFFEE_OPTIONS -o app/js -c app/coffee || echo 'No CoffeeScript folder to compile') && ( [ -e app.coffee ] && coffee -m $COFFEE_OPTIONS -c app.coffee || echo 'No CoffeeScript app to compile')",
"start": "npm run compile:app && node $NODE_APP_OPTIONS app.js", "start": "node $NODE_APP_OPTIONS app.js",
"test:acceptance:_run": "mocha --recursive --reporter spec --timeout 30000 --exit $@ test/acceptance/js", "test:acceptance:_run": "mocha --recursive --reporter spec --timeout 15000 --exit $@ test/acceptance/js",
"test:acceptance": "npm run compile:app && npm run compile:acceptance_tests && npm run test:acceptance:_run -- --grep=$MOCHA_GREP", "test:acceptance": "npm run test:acceptance:_run -- --grep=$MOCHA_GREP",
"test:unit:_run": "mocha --recursive --reporter spec --exit $@ test/unit/js", "test:unit:_run": "mocha --recursive --reporter spec --exit $@ test/unit/js",
"test:unit": "npm run compile:app && npm run compile:unit_tests && npm run test:unit:_run -- --grep=$MOCHA_GREP", "test:unit": "npm run test:unit:_run -- --grep=$MOCHA_GREP",
"compile:unit_tests": "[ ! -e test/unit/coffee ] && echo 'No unit tests to compile' || coffee -o test/unit/js -c test/unit/coffee", "compile:unit_tests": "[ ! -e test/unit/coffee ] && echo 'No unit tests to compile' || coffee -o test/unit/js -c test/unit/coffee",
"compile:acceptance_tests": "[ ! -e test/acceptance/coffee ] && echo 'No acceptance tests to compile' || coffee -o test/acceptance/js -c test/acceptance/coffee", "compile:acceptance_tests": "[ ! -e test/acceptance/coffee ] && echo 'No acceptance tests to compile' || coffee -o test/acceptance/js -c test/acceptance/coffee",
"compile:all": "npm run compile:app && npm run compile:unit_tests && npm run compile:acceptance_tests && npm run compile:smoke_tests", "compile:all": "npm run compile:app && npm run compile:unit_tests && npm run compile:acceptance_tests && npm run compile:smoke_tests",
"nodemon": "nodemon --config nodemon.json", "nodemon": "nodemon --config nodemon.json",
"compile:smoke_tests": "[ ! -e test/smoke/coffee ] && echo 'No smoke tests to compile' || coffee -o test/smoke/js -c test/smoke/coffee" "compile:smoke_tests": "[ ! -e test/smoke/coffee ] && echo 'No smoke tests to compile' || coffee -o test/smoke/js -c test/smoke/coffee",
"lint": "node_modules/.bin/eslint .",
"format": "node_modules/.bin/prettier-eslint '**/*.js' --list-different",
"format:fix": "node_modules/.bin/prettier-eslint '**/*.js' --write"
}, },
"version": "0.1.4", "version": "0.1.4",
"dependencies": { "dependencies": {
@ -39,7 +42,19 @@
"devDependencies": { "devDependencies": {
"bunyan": "^1.0.0", "bunyan": "^1.0.0",
"chai": "", "chai": "",
"eslint": "^6.0.1",
"eslint-config-prettier": "^6.0.0",
"eslint-config-standard": "^12.0.0",
"eslint-plugin-chai-expect": "^2.0.1",
"eslint-plugin-chai-friendly": "^0.4.1",
"eslint-plugin-import": "^2.18.0",
"eslint-plugin-mocha": "^5.3.0",
"eslint-plugin-node": "^9.1.0",
"eslint-plugin-prettier": "^3.1.0",
"eslint-plugin-promise": "^4.2.1",
"eslint-plugin-standard": "^4.0.0",
"mocha": "^4.1.0", "mocha": "^4.1.0",
"prettier-eslint-cli": "^5.0.0",
"sandboxed-module": "", "sandboxed-module": "",
"sinon": "^1.17.0" "sinon": "^1.17.0"
} }

86
services/spelling/test/stress/coffee/stressTest.coffee
View file

@ -1,86 +0,0 @@
# N requests in parallel
# send P correct words and Q incorrect words
# generate incorrect words by qq+random
async = require "async"
request = require "request"
fs = require "fs"
# created with
# aspell -d en dump master | aspell -l en expand | shuf -n 150000 > words.txt
WORDS = "words.txt"
wordlist = fs.readFileSync(WORDS).toString().split('\n').filter (w) ->
w.match(/^[a-z]+$/)
generateCorrectWords = (n) ->
words = []
N = if Math.random() > 0.5 then wordlist.length else 10
for i in [1 .. n]
j = Math.floor(N * Math.random())
words.push wordlist[j]
return words
generateIncorrectWords = (n) ->
words = []
N = wordlist.length
for i in [1 .. n]
j = Math.floor(N * Math.random())
words.push("qzxq" + wordlist[j])
return words
make_request = (correctWords, incorrectWords, callback) ->
correctSet = generateCorrectWords(correctWords)
incorrectSet = generateIncorrectWords(incorrectWords)
correctSet.push('constructor')
incorrectSet.push('qzxqfoofoofoo')
full = correctSet.concat incorrectSet
bad = []
for w, i in correctSet
bad[i] = false
for w, i in incorrectSet
bad[i+correctSet.length] = true
k = full.length
full.forEach (e, i) ->
j = Math.floor(k * Math.random())
[ full[i], full[j] ] = [ full[j], full[i] ]
[ bad[i], bad[j] ] = [ bad[j], bad[i] ]
expected = []
for tf, i in bad
if tf
expected.push {index: i, word: full[i]}
request.post 'http://localhost:3005/user/1/check', json:true, body: {words: full}, (err, req, body) ->
misspellings = body.misspellings
console.log JSON.stringify({full: full, misspellings: misspellings})
if expected.length != misspellings.length
console.log "ERROR: length mismatch", expected.length, misspellings.length
console.log full, bad
console.log 'expected', expected, 'mispellings', misspellings
for i in [0 .. Math.max(expected.length, misspellings.length)-1]
if expected[i].index != misspellings[i].index
console.log "ERROR", i, expected[i], misspellings[i], full[misspellings[i].index]
for m in misspellings
console.log full[m.index], "=>", m
process.exit()
callback("error")
else
for m,i in body.misspellings
if m.index != expected[i].index
console.log "ERROR AT RESULT", i, m, expected[i]
process.exit()
callback("error")
callback(null, full)
q = async.queue (task, callback) ->
setTimeout () ->
make_request task.correct, task.incorrect, callback
, Math.random() * 100
, 3
q.drain = () ->
console.log('all items have been processed');
for i in [0 .. 1000]
q.push({correct: Math.floor(30*Math.random()) + 1, incorrect: Math.floor(3*Math.random())})
# if Math.random() < 0.1
# else
# q.push({correct: Math.floor(100*Math.random()) + 1, incorrect: Math.floor(3*Math.random())})

162
services/spelling/test/stress/js/stressTest.js Normal file
View file

@ -0,0 +1,162 @@
/* eslint-disable */
// TODO: This file was created by bulk-decaffeinate.
// Sanity-check the conversion and remove this comment.
/*
* decaffeinate suggestions:
* DS101: Remove unnecessary use of Array.from
* DS102: Remove unnecessary code created because of implicit returns
* DS104: Avoid inline assignments
* DS202: Simplify dynamic range loops
* Full docs: https://github.com/decaffeinate/decaffeinate/blob/master/docs/suggestions.md
*/
// N requests in parallel
// send P correct words and Q incorrect words
// generate incorrect words by qq+random
const async = require('async')
const request = require('request')
const fs = require('fs')
// created with
// aspell -d en dump master | aspell -l en expand | shuf -n 150000 > words.txt
const WORDS = 'words.txt'
const wordlist = fs
.readFileSync(WORDS)
.toString()
.split('\n')
.filter(w => w.match(/^[a-z]+$/))
const generateCorrectWords = function(n) {
const words = []
const N = Math.random() > 0.5 ? wordlist.length : 10
for (
let i = 1, end = n, asc = 1 <= end;
asc ? i <= end : i >= end;
asc ? i++ : i--
) {
const j = Math.floor(N * Math.random())
words.push(wordlist[j])
}
return words
}
const generateIncorrectWords = function(n) {
const words = []
const N = wordlist.length
for (
let i = 1, end = n, asc = 1 <= end;
asc ? i <= end : i >= end;
asc ? i++ : i--
) {
const j = Math.floor(N * Math.random())
words.push(`qzxq${wordlist[j]}`)
}
return words
}
const make_request = function(correctWords, incorrectWords, callback) {
let i, j, w
let i1
let j1
const correctSet = generateCorrectWords(correctWords)
const incorrectSet = generateIncorrectWords(incorrectWords)
correctSet.push('constructor')
incorrectSet.push('qzxqfoofoofoo')
const full = correctSet.concat(incorrectSet)
const bad = []
for (j = 0, i = j; j < correctSet.length; j++, i = j) {
w = correctSet[i]
bad[i] = false
}
for (i1 = 0, i = i1; i1 < incorrectSet.length; i1++, i = i1) {
w = incorrectSet[i]
bad[i + correctSet.length] = true
}
const k = full.length
full.forEach(function(e, i) {
let ref
j = Math.floor(k * Math.random())
;[full[i], full[j]] = Array.from([full[j], full[i]])
return ([bad[i], bad[j]] = Array.from((ref = [bad[j], bad[i]]))), ref
})
const expected = []
for (j1 = 0, i = j1; j1 < bad.length; j1++, i = j1) {
const tf = bad[i]
if (tf) {
expected.push({ index: i, word: full[i] })
}
}
return request.post(
'http://localhost:3005/user/1/check',
{ json: true, body: { words: full } },
function(err, req, body) {
let m
const { misspellings } = body
console.log(JSON.stringify({ full, misspellings }))
if (expected.length !== misspellings.length) {
let asc, end
console.log(
'ERROR: length mismatch',
expected.length,
misspellings.length
)
console.log(full, bad)
console.log('expected', expected, 'mispellings', misspellings)
for (
i = 0,
end = Math.max(expected.length, misspellings.length) - 1,
asc = 0 <= end;
asc ? i <= end : i >= end;
asc ? i++ : i--
) {
if (expected[i].index !== misspellings[i].index) {
console.log(
'ERROR',
i,
expected[i],
misspellings[i],
full[misspellings[i].index]
)
}
}
for (m of Array.from(misspellings)) {
console.log(full[m.index], '=>', m)
}
process.exit()
callback('error')
} else {
for (i = 0; i < body.misspellings.length; i++) {
m = body.misspellings[i]
if (m.index !== expected[i].index) {
console.log('ERROR AT RESULT', i, m, expected[i])
process.exit()
callback('error')
}
}
}
return callback(null, full)
}
)
}
const q = async.queue(
(task, callback) =>
setTimeout(
() => make_request(task.correct, task.incorrect, callback),
Math.random() * 100
),
3
)
q.drain = () => console.log('all items have been processed')
for (let i = 0; i <= 1000; i++) {
q.push({
correct: Math.floor(30 * Math.random()) + 1,
incorrect: Math.floor(3 * Math.random())
})
}
// if Math.random() < 0.1
// else
// q.push({correct: Math.floor(100*Math.random()) + 1, incorrect: Math.floor(3*Math.random())})

0
services/spelling/test/stress/coffee/words.txt → services/spelling/test/stress/js/words.txt
View file

70
services/spelling/test/unit/coffee/ASpellTests.coffee
View file

@ -1,70 +0,0 @@
sinon = require 'sinon'
chai = require 'chai'
should = chai.should()
SandboxedModule = require('sandboxed-module')
assert = require("chai").assert
describe "ASpell", ->
beforeEach ->
@ASpell = SandboxedModule.require "../../../app/js/ASpell", requires:
"logger-sharelatex":
log:->
info:->
err:->
'metrics-sharelatex':
gauge:->
inc: ->
describe "a correctly spelled word", ->
beforeEach (done) ->
@ASpell.checkWords "en", ["word"], (error, @result) => done()
it "should not correct the word", ->
@result.length.should.equal 0
describe "a misspelled word", ->
beforeEach (done) ->
@ASpell.checkWords "en", ["bussines"], (error, @result) => done()
it "should correct the word", ->
@result.length.should.equal 1
@result[0].suggestions.indexOf("business").should.not.equal -1
describe "multiple words", ->
beforeEach (done) ->
@ASpell.checkWords "en", ["bussines", "word", "neccesary"], (error, @result) => done()
it "should correct the incorrect words", ->
@result[0].index.should.equal 0
@result[0].suggestions.indexOf("business").should.not.equal -1
@result[1].index.should.equal 2
@result[1].suggestions.indexOf("necessary").should.not.equal -1
describe "without a valid language", ->
beforeEach (done) ->
@ASpell.checkWords "notALang", ["banana"], (@error, @result) => done()
it "should return an error", ->
should.exist @error
describe "when there are no suggestions", ->
beforeEach (done) ->
@ASpell.checkWords "en", ["asdkfjalkdjfadhfkajsdhfashdfjhadflkjadhflajsd"], (@error, @result) => done()
it "should return a blank array", ->
@result.length.should.equal 1
assert.deepEqual @result[0].suggestions, []
describe "when the request times out", ->
beforeEach (done) ->
words = ("abcdefg" for i in [0..1000])
@ASpell.ASPELL_TIMEOUT = 1
@start = Date.now()
@ASpell.checkWords "en", words, (error, @result) => done()
# Note that this test fails on OS X, due to differing pipe behaviour
# on killing the child process. It can be tested successfully on Travis
# or the CI server.
it "should return in reasonable time", () ->
delta = Date.now()-@start
delta.should.be.below(@ASpell.ASPELL_TIMEOUT + 1000)

97
services/spelling/test/unit/coffee/LearnedWordsManagerTests.coffee
View file

@ -1,97 +0,0 @@
sinon = require('sinon')
chai = require 'chai'
expect = chai.expect
SandboxedModule = require('sandboxed-module')
modulePath = require('path').join __dirname, '../../../app/js/LearnedWordsManager'
assert = require("chai").assert
should = require("chai").should()
describe "LearnedWordsManager", ->
beforeEach ->
@token = "a6b3cd919ge"
@callback = sinon.stub()
@db =
spellingPreferences:
update: sinon.stub().callsArg(3)
@cache =
get:sinon.stub()
set:sinon.stub()
del:sinon.stub()
@LearnedWordsManager = SandboxedModule.require modulePath, requires:
"./DB" : @db
"./MongoCache":@cache
"logger-sharelatex":
log:->
err:->
info:->
'metrics-sharelatex': {timeAsyncMethod: sinon.stub(), inc: sinon.stub()}
describe "learnWord", ->
beforeEach ->
@word = "instanton"
@LearnedWordsManager.learnWord @token, @word, @callback
it "should insert the word in the word list in the database", ->
expect(
@db.spellingPreferences.update.calledWith({
token: @token
}, {
$push : learnedWords: @word
}, {
upsert: true
})
).to.equal true
it "should call the callback", ->
expect(@callback.called).to.equal true
describe "getLearnedWords", ->
beforeEach ->
@wordList = ["apples", "bananas", "pears"]
@db.spellingPreferences.findOne = (conditions, callback) =>
callback null, learnedWords: @wordList
sinon.spy @db.spellingPreferences, "findOne"
@LearnedWordsManager.getLearnedWords @token, @callback
it "should get the word list for the given user", ->
expect(
@db.spellingPreferences.findOne.calledWith token: @token
).to.equal true
it "should return the word list in the callback", ->
expect(@callback.calledWith null, @wordList).to.equal true
describe "caching the result", ->
it 'should use the cache first if it is primed', (done)->
@wordList = ["apples", "bananas", "pears"]
@cache.get.returns(@wordList)
@db.spellingPreferences.findOne = sinon.stub()
@LearnedWordsManager.getLearnedWords @token, (err, spellings)=>
@db.spellingPreferences.findOne.called.should.equal false
assert.deepEqual @wordList, spellings
done()
it 'should set the cache after hitting the db', (done)->
@wordList = ["apples", "bananas", "pears"]
@db.spellingPreferences.findOne = sinon.stub().callsArgWith(1, null, learnedWords: @wordList)
@LearnedWordsManager.getLearnedWords @token, (err, spellings)=>
@cache.set.calledWith(@token, @wordList).should.equal true
done()
it 'should break cache when update is called', (done)->
@word = "instanton"
@LearnedWordsManager.learnWord @token, @word, =>
@cache.del.calledWith(@token).should.equal true
done()
describe "deleteUsersLearnedWords", ->
beforeEach ->
@db.spellingPreferences.remove = sinon.stub().callsArgWith(1)
it "should get the word list for the given user", (done)->
@LearnedWordsManager.deleteUsersLearnedWords @token, =>
@db.spellingPreferences.remove.calledWith(token: @token).should.equal true
done()

119
services/spelling/test/unit/coffee/SpellingAPIManagerTests.coffee
View file

@ -1,119 +0,0 @@
sinon = require('sinon')
chai = require 'chai'
expect = chai.expect
chai.should()
SandboxedModule = require('sandboxed-module')
modulePath = require('path').join __dirname, '../../../app/js/SpellingAPIManager'
describe "SpellingAPIManager", ->
beforeEach ->
@token = "user-id-123"
@ASpell = {}
@learnedWords = ["lerned"]
@LearnedWordsManager =
getLearnedWords: sinon.stub().callsArgWith(1, null, @learnedWords)
learnWord: sinon.stub().callsArg(2)
@SpellingAPIManager = SandboxedModule.require modulePath, requires:
"./ASpell" : @ASpell
"./LearnedWordsManager" : @LearnedWordsManager
describe "runRequest", ->
beforeEach ->
@nonLearnedWords = ["some", "words", "htat", "are", "speled", "rong", "lerned"]
@allWords = @nonLearnedWords.concat(@learnedWords)
@misspellings = [
{ index: 2, suggestions: ["that"] }
{ index: 4, suggestions: ["spelled"] }
{ index: 5, suggestions: ["wrong", "ring"] }
{ index: 6, suggestions: ["learned"] }
]
@misspellingsWithoutLearnedWords = @misspellings.slice(0,3)
@ASpell.checkWords = (lang, word, callback) =>
callback null, @misspellings
sinon.spy @ASpell, "checkWords"
describe "with sensible JSON", ->
beforeEach (done) ->
@SpellingAPIManager.runRequest @token, words: @allWords, (error, @result) => done()
it "should return the words that are spelled incorrectly and not learned", ->
expect(@result.misspellings).to.deep.equal @misspellingsWithoutLearnedWords
describe "with a missing words array", ->
beforeEach (done) ->
@SpellingAPIManager.runRequest @token, {}, (@error, @result) => done()
it "should return an error", ->
expect(@error).to.exist
expect(@error).to.be.instanceof Error
expect(@error.message).to.equal "malformed JSON"
describe "with a missing token", ->
beforeEach (done) ->
@SpellingAPIManager.runRequest null, words: @allWords, (@error, @result) => done()
it "should spell check without using any learned words", ->
@LearnedWordsManager.getLearnedWords.called.should.equal false
describe "without a language", ->
beforeEach (done) ->
@SpellingAPIManager.runRequest @token, words: @allWords, (error, @result) => done()
it "should use en as the default", ->
@ASpell.checkWords.calledWith("en").should.equal true
describe "with a language", ->
beforeEach (done) ->
@SpellingAPIManager.runRequest @token, {
words: @allWords
language: @language = "fr"
}, (error, @result) => done()
it "should use the language", ->
@ASpell.checkWords.calledWith(@language).should.equal true
describe "with a very large collection of words", ->
beforeEach (done) ->
@manyWords = ("word" for i in [1..100000])
@SpellingAPIManager.runRequest @token, words: @manyWords, (error, @result) => done()
it "should truncate to 10,000 words", ->
@ASpell.checkWords.calledWith(sinon.match.any, @manyWords.slice(0, 10000)).should.equal true
describe 'with words from the whitelist', ->
beforeEach (done) ->
@whitelistWord = @SpellingAPIManager.whitelist[0]
@words = ["One", "Two", @whitelistWord]
@SpellingAPIManager.runRequest @token, words: @words, (error, @result) => done()
it 'should ignore the white-listed word', ->
expect(@result.misspellings.length).to.equal @misspellings.length-1
describe "learnWord", ->
describe "without a token", ->
beforeEach (done) -> @SpellingAPIManager.learnWord null, word: "banana", (@error) => done()
it "should return an error", ->
expect(@error).to.exist
expect(@error).to.be.instanceof Error
expect(@error.message).to.equal "no token provided"
describe "without a word", ->
beforeEach (done) -> @SpellingAPIManager.learnWord @token, {}, (@error) => done()
it "should return an error", ->
expect(@error).to.exist
expect(@error).to.be.instanceof Error
expect(@error.message).to.equal "malformed JSON"
describe "with a word and a token", ->
beforeEach (done) ->
@word = "banana"
@SpellingAPIManager.learnWord @token, word: @word, (@error) => done()
it "should call LearnedWordsManager.learnWord", ->
@LearnedWordsManager.learnWord.calledWith(@token, @word).should.equal true

145
services/spelling/test/unit/js/ASpellTests.js Normal file
View file

@ -0,0 +1,145 @@
/* eslint-disable
handle-callback-err,
no-undef
*/
// TODO: This file was created by bulk-decaffeinate.
// Sanity-check the conversion and remove this comment.
/*
* decaffeinate suggestions:
* DS102: Remove unnecessary code created because of implicit returns
* Full docs: https://github.com/decaffeinate/decaffeinate/blob/master/docs/suggestions.md
*/
const chai = require('chai')
const should = chai.should()
const SandboxedModule = require('sandboxed-module')
const { assert } = require('chai')
describe('ASpell', function() {
beforeEach(function() {
return (this.ASpell = SandboxedModule.require('../../../app/js/ASpell', {
requires: {
'logger-sharelatex': {
log() {},
info() {},
err() {}
},
'metrics-sharelatex': {
gauge() {},
inc() {}
}
}
}))
})
describe('a correctly spelled word', function() {
beforeEach(function(done) {
return this.ASpell.checkWords('en', ['word'], (error, result) => {
this.result = result
return done()
})
})
return it('should not correct the word', function() {
return this.result.length.should.equal(0)
})
})
describe('a misspelled word', function() {
beforeEach(function(done) {
return this.ASpell.checkWords('en', ['bussines'], (error, result) => {
this.result = result
return done()
})
})
return it('should correct the word', function() {
this.result.length.should.equal(1)
return this.result[0].suggestions.indexOf('business').should.not.equal(-1)
})
})
describe('multiple words', function() {
beforeEach(function(done) {
return this.ASpell.checkWords(
'en',
['bussines', 'word', 'neccesary'],
(error, result) => {
this.result = result
return done()
}
)
})
return it('should correct the incorrect words', function() {
this.result[0].index.should.equal(0)
this.result[0].suggestions.indexOf('business').should.not.equal(-1)
this.result[1].index.should.equal(2)
return this.result[1].suggestions
.indexOf('necessary')
.should.not.equal(-1)
})
})
describe('without a valid language', function() {
beforeEach(function(done) {
return this.ASpell.checkWords('notALang', ['banana'], (error, result) => {
this.error = error
this.result = result
return done()
})
})
return it('should return an error', function() {
return should.exist(this.error)
})
})
describe('when there are no suggestions', function() {
beforeEach(function(done) {
return this.ASpell.checkWords(
'en',
['asdkfjalkdjfadhfkajsdhfashdfjhadflkjadhflajsd'],
(error, result) => {
this.error = error
this.result = result
return done()
}
)
})
return it('should return a blank array', function() {
this.result.length.should.equal(1)
return assert.deepEqual(this.result[0].suggestions, [])
})
})
return describe('when the request times out', function() {
beforeEach(function(done) {
const words = __range__(0, 1000, true).map(i => 'abcdefg')
this.ASpell.ASPELL_TIMEOUT = 1
this.start = Date.now()
return this.ASpell.checkWords('en', words, (error, result) => {
this.result = result
return done()
})
})
// Note that this test fails on OS X, due to differing pipe behaviour
// on killing the child process. It can be tested successfully on Travis
// or the CI server.
return it('should return in reasonable time', function() {
const delta = Date.now() - this.start
return delta.should.be.below(this.ASpell.ASPELL_TIMEOUT + 1000)
})
})
})
function __range__(left, right, inclusive) {
let range = []
let ascending = left < right
let end = !inclusive ? right : ascending ? right + 1 : right - 1
for (let i = left; ascending ? i < end : i > end; ascending ? i++ : i--) {
range.push(i)
}
return range
}

163
services/spelling/test/unit/js/LearnedWordsManagerTests.js Normal file
View file

@ -0,0 +1,163 @@
/* eslint-disable
handle-callback-err,
no-undef
*/
// TODO: This file was created by bulk-decaffeinate.
// Sanity-check the conversion and remove this comment.
/*
* decaffeinate suggestions:
* DS102: Remove unnecessary code created because of implicit returns
* Full docs: https://github.com/decaffeinate/decaffeinate/blob/master/docs/suggestions.md
*/
const sinon = require('sinon')
const chai = require('chai')
const { expect } = chai
const SandboxedModule = require('sandboxed-module')
const modulePath = require('path').join(
__dirname,
'../../../app/js/LearnedWordsManager'
)
const { assert } = require('chai')
describe('LearnedWordsManager', function() {
beforeEach(function() {
this.token = 'a6b3cd919ge'
this.callback = sinon.stub()
this.db = {
spellingPreferences: {
update: sinon.stub().callsArg(3)
}
}
this.cache = {
get: sinon.stub(),
set: sinon.stub(),
del: sinon.stub()
}
return (this.LearnedWordsManager = SandboxedModule.require(modulePath, {
requires: {
'./DB': this.db,
'./MongoCache': this.cache,
'logger-sharelatex': {
log() {},
err() {},
info() {}
},
'metrics-sharelatex': {
timeAsyncMethod: sinon.stub(),
inc: sinon.stub()
}
}
}))
})
describe('learnWord', function() {
beforeEach(function() {
this.word = 'instanton'
return this.LearnedWordsManager.learnWord(
this.token,
this.word,
this.callback
)
})
it('should insert the word in the word list in the database', function() {
return expect(
this.db.spellingPreferences.update.calledWith(
{
token: this.token
},
{
$push: { learnedWords: this.word }
},
{
upsert: true
}
)
).to.equal(true)
})
return it('should call the callback', function() {
return expect(this.callback.called).to.equal(true)
})
})
describe('getLearnedWords', function() {
beforeEach(function() {
this.wordList = ['apples', 'bananas', 'pears']
this.db.spellingPreferences.findOne = (conditions, callback) => {
return callback(null, { learnedWords: this.wordList })
}
sinon.spy(this.db.spellingPreferences, 'findOne')
return this.LearnedWordsManager.getLearnedWords(this.token, this.callback)
})
it('should get the word list for the given user', function() {
return expect(
this.db.spellingPreferences.findOne.calledWith({ token: this.token })
).to.equal(true)
})
return it('should return the word list in the callback', function() {
return expect(this.callback.calledWith(null, this.wordList)).to.equal(
true
)
})
})
describe('caching the result', function() {
it('should use the cache first if it is primed', function(done) {
this.wordList = ['apples', 'bananas', 'pears']
this.cache.get.returns(this.wordList)
this.db.spellingPreferences.findOne = sinon.stub()
return this.LearnedWordsManager.getLearnedWords(
this.token,
(err, spellings) => {
this.db.spellingPreferences.findOne.called.should.equal(false)
assert.deepEqual(this.wordList, spellings)
return done()
}
)
})
it('should set the cache after hitting the db', function(done) {
this.wordList = ['apples', 'bananas', 'pears']
this.db.spellingPreferences.findOne = sinon
.stub()
.callsArgWith(1, null, { learnedWords: this.wordList })
return this.LearnedWordsManager.getLearnedWords(
this.token,
(err, spellings) => {
this.cache.set
.calledWith(this.token, this.wordList)
.should.equal(true)
return done()
}
)
})
return it('should break cache when update is called', function(done) {
this.word = 'instanton'
return this.LearnedWordsManager.learnWord(this.token, this.word, () => {
this.cache.del.calledWith(this.token).should.equal(true)
return done()
})
})
})
return describe('deleteUsersLearnedWords', function() {
beforeEach(function() {
return (this.db.spellingPreferences.remove = sinon.stub().callsArgWith(1))
})
return it('should get the word list for the given user', function(done) {
return this.LearnedWordsManager.deleteUsersLearnedWords(
this.token,
() => {
this.db.spellingPreferences.remove
.calledWith({ token: this.token })
.should.equal(true)
return done()
}
)
})
})
})

271
services/spelling/test/unit/js/SpellingAPIManagerTests.js Normal file
View file

@ -0,0 +1,271 @@
/* eslint-disable
handle-callback-err,
no-undef
*/
// TODO: This file was created by bulk-decaffeinate.
// Sanity-check the conversion and remove this comment.
/*
* decaffeinate suggestions:
* DS102: Remove unnecessary code created because of implicit returns
* Full docs: https://github.com/decaffeinate/decaffeinate/blob/master/docs/suggestions.md
*/
const sinon = require('sinon')
const chai = require('chai')
const { expect } = chai
chai.should()
const SandboxedModule = require('sandboxed-module')
const modulePath = require('path').join(
__dirname,
'../../../app/js/SpellingAPIManager'
)
describe('SpellingAPIManager', function() {
beforeEach(function() {
this.token = 'user-id-123'
this.ASpell = {}
this.learnedWords = ['lerned']
this.LearnedWordsManager = {
getLearnedWords: sinon.stub().callsArgWith(1, null, this.learnedWords),
learnWord: sinon.stub().callsArg(2)
}
return (this.SpellingAPIManager = SandboxedModule.require(modulePath, {
requires: {
'./ASpell': this.ASpell,
'./LearnedWordsManager': this.LearnedWordsManager
}
}))
})
describe('runRequest', function() {
beforeEach(function() {
this.nonLearnedWords = [
'some',
'words',
'htat',
'are',
'speled',
'rong',
'lerned'
]
this.allWords = this.nonLearnedWords.concat(this.learnedWords)
this.misspellings = [
{ index: 2, suggestions: ['that'] },
{ index: 4, suggestions: ['spelled'] },
{ index: 5, suggestions: ['wrong', 'ring'] },
{ index: 6, suggestions: ['learned'] }
]
this.misspellingsWithoutLearnedWords = this.misspellings.slice(0, 3)
this.ASpell.checkWords = (lang, word, callback) => {
return callback(null, this.misspellings)
}
return sinon.spy(this.ASpell, 'checkWords')
})
describe('with sensible JSON', function() {
beforeEach(function(done) {
return this.SpellingAPIManager.runRequest(
this.token,
{ words: this.allWords },
(error, result) => {
this.result = result
return done()
}
)
})
return it('should return the words that are spelled incorrectly and not learned', function() {
return expect(this.result.misspellings).to.deep.equal(
this.misspellingsWithoutLearnedWords
)
})
})
describe('with a missing words array', function() {
beforeEach(function(done) {
return this.SpellingAPIManager.runRequest(
this.token,
{},
(error, result) => {
this.error = error
this.result = result
return done()
}
)
})
return it('should return an error', function() {
expect(this.error).to.exist
expect(this.error).to.be.instanceof(Error)
return expect(this.error.message).to.equal('malformed JSON')
})
})
describe('with a missing token', function() {
beforeEach(function(done) {
return this.SpellingAPIManager.runRequest(
null,
{ words: this.allWords },
(error, result) => {
this.error = error
this.result = result
return done()
}
)
})
return it('should spell check without using any learned words', function() {
return this.LearnedWordsManager.getLearnedWords.called.should.equal(
false
)
})
})
describe('without a language', function() {
beforeEach(function(done) {
return this.SpellingAPIManager.runRequest(
this.token,
{ words: this.allWords },
(error, result) => {
this.result = result
return done()
}
)
})
return it('should use en as the default', function() {
return this.ASpell.checkWords.calledWith('en').should.equal(true)
})
})
describe('with a language', function() {
beforeEach(function(done) {
return this.SpellingAPIManager.runRequest(
this.token,
{
words: this.allWords,
language: (this.language = 'fr')
},
(error, result) => {
this.result = result
return done()
}
)
})
return it('should use the language', function() {
return this.ASpell.checkWords
.calledWith(this.language)
.should.equal(true)
})
})
describe('with a very large collection of words', function() {
beforeEach(function(done) {
this.manyWords = __range__(1, 100000, true).map(i => 'word')
return this.SpellingAPIManager.runRequest(
this.token,
{ words: this.manyWords },
(error, result) => {
this.result = result
return done()
}
)
})
return it('should truncate to 10,000 words', function() {
return this.ASpell.checkWords
.calledWith(sinon.match.any, this.manyWords.slice(0, 10000))
.should.equal(true)
})
})
return describe('with words from the whitelist', function() {
beforeEach(function(done) {
this.whitelistWord = this.SpellingAPIManager.whitelist[0]
this.words = ['One', 'Two', this.whitelistWord]
return this.SpellingAPIManager.runRequest(
this.token,
{ words: this.words },
(error, result) => {
this.result = result
return done()
}
)
})
return it('should ignore the white-listed word', function() {
return expect(this.result.misspellings.length).to.equal(
this.misspellings.length - 1
)
})
})
})
return describe('learnWord', function() {
describe('without a token', function() {
beforeEach(function(done) {
return this.SpellingAPIManager.learnWord(
null,
{ word: 'banana' },
error => {
this.error = error
return done()
}
)
})
return it('should return an error', function() {
expect(this.error).to.exist
expect(this.error).to.be.instanceof(Error)
return expect(this.error.message).to.equal('no token provided')
})
})
describe('without a word', function() {
beforeEach(function(done) {
return this.SpellingAPIManager.learnWord(this.token, {}, error => {
this.error = error
return done()
})
})
return it('should return an error', function() {
expect(this.error).to.exist
expect(this.error).to.be.instanceof(Error)
return expect(this.error.message).to.equal('malformed JSON')
})
})
return describe('with a word and a token', function() {
beforeEach(function(done) {
this.word = 'banana'
return this.SpellingAPIManager.learnWord(
this.token,
{ word: this.word },
error => {
this.error = error
return done()
}
)
})
return it('should call LearnedWordsManager.learnWord', function() {
return this.LearnedWordsManager.learnWord
.calledWith(this.token, this.word)
.should.equal(true)
})
})
})
})
function __range__(left, right, inclusive) {
let range = []
let ascending = left < right
let end = !inclusive ? right : ascending ? right + 1 : right - 1
for (let i = left; ascending ? i < end : i > end; ascending ? i++ : i--) {
range.push(i)
}
return range
}