From 562f14455c23a4078cbdef4f042897c41a30c094 Mon Sep 17 00:00:00 2001 From: Miguel Serrano Date: Wed, 3 Jun 2020 10:49:09 +0200 Subject: [PATCH 1/3] updated build scripts --- services/spelling/Dockerfile | 2 +- services/spelling/Makefile | 42 +++++++++++++++++++++++------ services/spelling/buildscript.txt | 2 +- services/spelling/package-lock.json | 6 ++--- services/spelling/package.json | 8 +++--- 5 files changed, 43 insertions(+), 17 deletions(-) diff --git a/services/spelling/Dockerfile b/services/spelling/Dockerfile index dfa61290ce..0cd14f8f02 100644 --- a/services/spelling/Dockerfile +++ b/services/spelling/Dockerfile @@ -13,7 +13,7 @@ FROM base as app #wildcard as some files may not be in all repos COPY package*.json npm-shrink*.json /app/ -RUN npm install --quiet +RUN npm ci --quiet COPY . /app diff --git a/services/spelling/Makefile b/services/spelling/Makefile index 81fbc1922d..13966a0c1e 100644 --- a/services/spelling/Makefile +++ b/services/spelling/Makefile @@ -5,6 +5,8 @@ BUILD_NUMBER ?= local BRANCH_NAME ?= $(shell git rev-parse --abbrev-ref HEAD) PROJECT_NAME = spelling +BUILD_DIR_NAME = $(shell pwd | xargs basename | tr -cd '[a-zA-Z0-9_.\-]') + DOCKER_COMPOSE_FLAGS ?= -f docker-compose.yml DOCKER_COMPOSE := BUILD_NUMBER=$(BUILD_NUMBER) \ BRANCH_NAME=$(BRANCH_NAME) \ @@ -12,6 +14,12 @@ DOCKER_COMPOSE := BUILD_NUMBER=$(BUILD_NUMBER) \ MOCHA_GREP=${MOCHA_GREP} \ docker-compose ${DOCKER_COMPOSE_FLAGS} +DOCKER_COMPOSE_TEST_ACCEPTANCE = \ + COMPOSE_PROJECT_NAME=test_acceptance_$(BUILD_DIR_NAME) $(DOCKER_COMPOSE) + +DOCKER_COMPOSE_TEST_UNIT = \ + COMPOSE_PROJECT_NAME=test_unit_$(BUILD_DIR_NAME) $(DOCKER_COMPOSE) + clean: docker rmi ci/$(PROJECT_NAME):$(BRANCH_NAME)-$(BUILD_NUMBER) docker rmi gcr.io/overleaf-ops/$(PROJECT_NAME):$(BRANCH_NAME)-$(BUILD_NUMBER) @@ -28,23 +36,41 @@ lint: test: format lint test_unit test_acceptance test_unit: - @[ ! -d test/unit ] && echo "spelling has no unit tests" || $(DOCKER_COMPOSE) run --rm test_unit +ifneq (,$(wildcard test/unit)) + $(DOCKER_COMPOSE_TEST_UNIT) run --rm test_unit + $(MAKE) test_unit_clean +endif -test_acceptance: test_clean test_acceptance_pre_run test_acceptance_run +test_clean: test_unit_clean +test_unit_clean: +ifneq (,$(wildcard test/unit)) + $(DOCKER_COMPOSE_TEST_UNIT) down -v -t 0 +endif -test_acceptance_debug: test_clean test_acceptance_pre_run test_acceptance_run_debug +test_acceptance: test_acceptance_clean test_acceptance_pre_run test_acceptance_run + $(MAKE) test_acceptance_clean + +test_acceptance_debug: test_acceptance_clean test_acceptance_pre_run test_acceptance_run_debug + $(MAKE) test_acceptance_clean test_acceptance_run: - @[ ! -d test/acceptance ] && echo "spelling has no acceptance tests" || $(DOCKER_COMPOSE) run --rm test_acceptance +ifneq (,$(wildcard test/acceptance)) + $(DOCKER_COMPOSE_TEST_ACCEPTANCE) run --rm test_acceptance +endif test_acceptance_run_debug: - @[ ! -d test/acceptance ] && echo "spelling has no acceptance tests" || $(DOCKER_COMPOSE) run -p 127.0.0.9:19999:19999 --rm test_acceptance npm run test:acceptance -- --inspect=0.0.0.0:19999 --inspect-brk +ifneq (,$(wildcard test/acceptance)) + $(DOCKER_COMPOSE_TEST_ACCEPTANCE) run -p 127.0.0.9:19999:19999 --rm test_acceptance npm run test:acceptance -- --inspect=0.0.0.0:19999 --inspect-brk +endif -test_clean: - $(DOCKER_COMPOSE) down -v -t 0 +test_clean: test_acceptance_clean +test_acceptance_clean: + $(DOCKER_COMPOSE_TEST_ACCEPTANCE) down -v -t 0 test_acceptance_pre_run: - @[ ! -f test/acceptance/js/scripts/pre-run ] && echo "spelling has no pre acceptance tests task" || $(DOCKER_COMPOSE) run --rm test_acceptance test/acceptance/js/scripts/pre-run +ifneq (,$(wildcard test/acceptance/js/scripts/pre-run)) + $(DOCKER_COMPOSE_TEST_ACCEPTANCE) run --rm test_acceptance test/acceptance/js/scripts/pre-run +endif build: docker build --pull --tag ci/$(PROJECT_NAME):$(BRANCH_NAME)-$(BUILD_NUMBER) \ diff --git a/services/spelling/buildscript.txt b/services/spelling/buildscript.txt index 4df27b4801..922482948a 100644 --- a/services/spelling/buildscript.txt +++ b/services/spelling/buildscript.txt @@ -8,4 +8,4 @@ spelling --language=es --node-version=10.19.0 --public-repo=False ---script-version=2.0.0 +--script-version=2.2.0 diff --git a/services/spelling/package-lock.json b/services/spelling/package-lock.json index 550cf083d1..b67de884ae 100644 --- a/services/spelling/package-lock.json +++ b/services/spelling/package-lock.json @@ -5827,9 +5827,9 @@ "integrity": "sha1-RPoWGwGHuVSd2Eu5GAL5vYOFzWo=" }, "sandboxed-module": { - "version": "2.0.3", - "resolved": "https://registry.npmjs.org/sandboxed-module/-/sandboxed-module-2.0.3.tgz", - "integrity": "sha1-x+VFkzm7y6KMUwPusz9ug4e/upY=", + "version": "2.0.4", + "resolved": "https://registry.npmjs.org/sandboxed-module/-/sandboxed-module-2.0.4.tgz", + "integrity": "sha512-AwEPOdO8mg/wJjr876yCHP2DHqVN0MaggEXhp6IIf3bcI5cYoQl9QrrCHSrvToHjvdEiS5x4TVZRgjD2bEmNTA==", "requires": { "require-like": "0.1.2", "stack-trace": "0.0.9" diff --git a/services/spelling/package.json b/services/spelling/package.json index 36335a1865..1e6312e760 100644 --- a/services/spelling/package.json +++ b/services/spelling/package.json @@ -42,13 +42,13 @@ "bunyan": "^1.8.12", "chai": "", "eslint": "^6.8.0", - "eslint-config-prettier": "^6.10.1", - "eslint-config-standard": "^14.1.1", + "eslint-config-prettier": "^6.10.0", + "eslint-config-standard": "^14.1.0", "eslint-plugin-chai-expect": "^2.1.0", "eslint-plugin-chai-friendly": "^0.5.0", - "eslint-plugin-import": "^2.20.2", + "eslint-plugin-import": "^2.20.1", "eslint-plugin-mocha": "^6.3.0", - "eslint-plugin-node": "^11.1.0", + "eslint-plugin-node": "^11.0.0", "eslint-plugin-prettier": "^3.1.2", "eslint-plugin-promise": "^4.2.1", "eslint-plugin-standard": "^4.0.1", From 803ed1607cf5829d73a8ada622a2ffc5aeacd7d7 Mon Sep 17 00:00:00 2001 From: Miguel Serrano Date: Wed, 3 Jun 2020 10:52:36 +0200 Subject: [PATCH 2/3] format:fix --- services/spelling/app/js/ASpell.js | 6 +-- services/spelling/config/settings.defaults.js | 6 +-- .../spelling/test/acceptance/js/CheckTest.js | 38 +++++++++---------- .../test/acceptance/js/HealthCheckTest.js | 2 +- services/spelling/test/acceptance/js/Init.js | 4 +- .../spelling/test/acceptance/js/StatusTest.js | 2 +- .../test/acceptance/js/helpers/request.js | 3 +- services/spelling/test/unit/js/ASpellTests.js | 8 ++-- 8 files changed, 35 insertions(+), 34 deletions(-) diff --git a/services/spelling/app/js/ASpell.js b/services/spelling/app/js/ASpell.js index 04367233cb..4f2b251d25 100644 --- a/services/spelling/app/js/ASpell.js +++ b/services/spelling/app/js/ASpell.js @@ -89,7 +89,7 @@ class ASpellRunner { // update the cache after processing all words, to avoid cache // changing while we use it - for (let k in addToCache) { + for (const k in addToCache) { const v = addToCache[k] cache.set(k, v) } @@ -109,7 +109,7 @@ class ASpellRunner { getSuggestions(language, output) { const lines = output.split('\n') const suggestions = {} - for (let line of Array.from(lines)) { + for (const line of Array.from(lines)) { var parts, word if (line[0] === '&') { // Suggestions found @@ -149,7 +149,7 @@ class ASpellRunner { callback = () => {} } const newWord = {} - for (let word of Array.from(words)) { + for (const word of Array.from(words)) { if (!newWord[word] && !cache.has(language + ':' + word)) { newWord[word] = true } diff --git a/services/spelling/config/settings.defaults.js b/services/spelling/config/settings.defaults.js index 2f3bc541c3..03e22c010c 100644 --- a/services/spelling/config/settings.defaults.js +++ b/services/spelling/config/settings.defaults.js @@ -4,14 +4,14 @@ module.exports = { internal: { spelling: { port: 3005, - host: process.env['LISTEN_ADDRESS'] || 'localhost' + host: process.env.LISTEN_ADDRESS || 'localhost' } }, mongo: { url: - process.env['MONGO_CONNECTION_STRING'] || - `mongodb://${process.env['MONGO_HOST'] || 'localhost'}/sharelatex` + process.env.MONGO_CONNECTION_STRING || + `mongodb://${process.env.MONGO_HOST || 'localhost'}/sharelatex` }, cacheDir: Path.resolve('cache'), diff --git a/services/spelling/test/acceptance/js/CheckTest.js b/services/spelling/test/acceptance/js/CheckTest.js index 160f0de457..943ee085e2 100644 --- a/services/spelling/test/acceptance/js/CheckTest.js +++ b/services/spelling/test/acceptance/js/CheckTest.js @@ -16,15 +16,15 @@ describe('checking words', function() { let response describe('on successful response', function() { - beforeEach(async function () { + beforeEach(async function() { response = await checkWord(['anather']) }) - it('should return status 200', async function () { + it('should return status 200', async function() { expect(response.statusCode).to.equal(200) }) - it('should return the list of misspellings', async function () { + it('should return the list of misspellings', async function() { const body = JSON.parse(response.body) expect(body).to.deep.equal({ misspellings: [{ index: 0, suggestions: ['anther', 'another'] }] @@ -33,22 +33,22 @@ describe('checking words', function() { }) describe('when multiple words are submitted', function() { - beforeEach(async function () { + beforeEach(async function() { response = await checkWord(['anather', 'anather', 'theorie']) }) - it('should return the misspellings for all the words', async function () { + it('should return the misspellings for all the words', async function() { const body = JSON.parse(response.body) expect(body.misspellings.length).to.equal(3) }) - it('should have misspelling suggestions with consecutive indexes', async function () { + it('should have misspelling suggestions with consecutive indexes', async function() { const body = JSON.parse(response.body) const indexes = body.misspellings.map(mspl => mspl.index) expect(indexes).to.deep.equal([0, 1, 2]) }) - it('should return identical suggestions for the same entry', async function () { + it('should return identical suggestions for the same entry', async function() { const body = JSON.parse(response.body) expect(body.misspellings[0].suggestions).to.deep.equal( body.misspellings[1].suggestions @@ -57,20 +57,20 @@ describe('checking words', function() { }) describe('when a very long list of words if submitted', function() { - beforeEach(async function () { - let words = [] + beforeEach(async function() { + const words = [] for (let i = 0; i <= 20000; i++) { words.push('anather') } response = await checkWord(words) }) - it('should return misspellings for the first 10K results only', async function () { + it('should return misspellings for the first 10K results only', async function() { const body = JSON.parse(response.body) expect(body.misspellings.length).to.equal(10000) }) - it('should have misspelling suggestions with consecutive indexes', async function () { + it('should have misspelling suggestions with consecutive indexes', async function() { const body = JSON.parse(response.body) const indexList = body.misspellings.map(mspl => mspl.index) expect(indexList.length).to.equal(10000) // avoid testing over an incorrect array @@ -81,20 +81,20 @@ describe('checking words', function() { }) describe('when a very long list of words with utf8 responses', function() { - beforeEach(async function () { - let words = [] + beforeEach(async function() { + const words = [] for (let i = 0; i <= 20000; i++) { words.push('anéther') } response = await checkWord(words, 'bg') // use Bulgarian to generate utf8 response }) - it('should return misspellings for the first 10K results only', async function () { + it('should return misspellings for the first 10K results only', async function() { const body = JSON.parse(response.body) expect(body.misspellings.length).to.equal(10000) }) - it('should have misspelling suggestions with consecutive indexes', async function () { + it('should have misspelling suggestions with consecutive indexes', async function() { const body = JSON.parse(response.body) const indexList = body.misspellings.map(mspl => mspl.index) expect(indexList.length).to.equal(10000) // avoid testing over an incorrect array @@ -105,22 +105,22 @@ describe('checking words', function() { }) describe('when multiple words with utf8 are submitted', function() { - beforeEach(async function () { + beforeEach(async function() { response = await checkWord(['mneá', 'meniésn', 'meônoi', 'mneá'], 'pt_BR') }) - it('should return the misspellings for all the words', async function () { + it('should return the misspellings for all the words', async function() { const body = JSON.parse(response.body) expect(body.misspellings.length).to.equal(4) }) - it('should have misspelling suggestions with consecutive indexes', async function () { + it('should have misspelling suggestions with consecutive indexes', async function() { const body = JSON.parse(response.body) const indexes = body.misspellings.map(mspl => mspl.index) expect(indexes).to.deep.equal([0, 1, 2, 3]) }) - it('should return identical suggestions for the same entry', async function () { + it('should return identical suggestions for the same entry', async function() { const body = JSON.parse(response.body) expect(body.misspellings[0].suggestions).to.deep.equal( body.misspellings[3].suggestions diff --git a/services/spelling/test/acceptance/js/HealthCheckTest.js b/services/spelling/test/acceptance/js/HealthCheckTest.js index 897dc43fa9..02c18a5938 100644 --- a/services/spelling/test/acceptance/js/HealthCheckTest.js +++ b/services/spelling/test/acceptance/js/HealthCheckTest.js @@ -2,7 +2,7 @@ const { expect } = require('chai') const request = require('./helpers/request') describe('/health_check', function() { - it('should return 200', async function () { + it('should return 200', async function() { const response = await request.get('/health_check') expect(response.statusCode).to.equal(200) }) diff --git a/services/spelling/test/acceptance/js/Init.js b/services/spelling/test/acceptance/js/Init.js index 4de82a6524..d52205c85f 100644 --- a/services/spelling/test/acceptance/js/Init.js +++ b/services/spelling/test/acceptance/js/Init.js @@ -1,4 +1,6 @@ const App = require('../../../app.js') const { PORT } = require('./helpers/request') -before(function(done) { return App.listen(PORT, 'localhost', done); }) +before(function(done) { + return App.listen(PORT, 'localhost', done) +}) diff --git a/services/spelling/test/acceptance/js/StatusTest.js b/services/spelling/test/acceptance/js/StatusTest.js index 9f006eebf1..5736bbb13a 100644 --- a/services/spelling/test/acceptance/js/StatusTest.js +++ b/services/spelling/test/acceptance/js/StatusTest.js @@ -2,7 +2,7 @@ const { expect } = require('chai') const request = require('./helpers/request') describe('/status', function() { - it('should return 200', async function () { + it('should return 200', async function() { const response = await request.get('/health_check') expect(response.statusCode).to.equal(200) }) diff --git a/services/spelling/test/acceptance/js/helpers/request.js b/services/spelling/test/acceptance/js/helpers/request.js index 538e29f1b6..e982a4b98c 100644 --- a/services/spelling/test/acceptance/js/helpers/request.js +++ b/services/spelling/test/acceptance/js/helpers/request.js @@ -2,8 +2,7 @@ const { promisify } = require('util') const PORT = 3005 -const BASE_URL = `http://${process.env['HTTP_TEST_HOST'] || - 'localhost'}:${PORT}` +const BASE_URL = `http://${process.env.HTTP_TEST_HOST || 'localhost'}:${PORT}` const request = require('request').defaults({ baseUrl: BASE_URL, diff --git a/services/spelling/test/unit/js/ASpellTests.js b/services/spelling/test/unit/js/ASpellTests.js index 87b659d7ec..05180d5651 100644 --- a/services/spelling/test/unit/js/ASpellTests.js +++ b/services/spelling/test/unit/js/ASpellTests.js @@ -30,7 +30,7 @@ describe('ASpell', function() { } })) }) - afterEach(function () { + afterEach(function() { clearInterval(this.ASpell.cacheDump) }) @@ -138,9 +138,9 @@ describe('ASpell', function() { }) function __range__(left, right, inclusive) { - let range = [] - let ascending = left < right - let end = !inclusive ? right : ascending ? right + 1 : right - 1 + const range = [] + const ascending = left < right + const end = !inclusive ? right : ascending ? right + 1 : right - 1 for (let i = left; ascending ? i < end : i > end; ascending ? i++ : i--) { range.push(i) } From b91e59e8a76ae5e2e6f90bc678e0411a25e45f09 Mon Sep 17 00:00:00 2001 From: Miguel Serrano Date: Wed, 3 Jun 2020 11:00:35 +0200 Subject: [PATCH 3/3] fixed linting --- services/spelling/app/js/ASpellWorker.js | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/services/spelling/app/js/ASpellWorker.js b/services/spelling/app/js/ASpellWorker.js index c4ead15083..834917b755 100644 --- a/services/spelling/app/js/ASpellWorker.js +++ b/services/spelling/app/js/ASpellWorker.js @@ -90,7 +90,10 @@ class ASpellWorker { ) this.callback = null } else { - logger.warn({ error: err, ...errInfo }, 'aspell worker error') + logger.warn( + Object.assign({ error: err }, errInfo), + 'aspell worker error' + ) } }) this.pipe.stdin.on('error', err => { @@ -119,10 +122,7 @@ class ASpellWorker { this.callback = null } else { logger.warn( - { - error: err, - ...errInfo - }, + Object.assign({ error: err }, errInfo), 'aspell worker error on stdin' ) } @@ -134,7 +134,7 @@ class ASpellWorker { this.pipe.stdout.on('data', data => { // We receive the language code from Aspell as the end of data marker in // the data. The input is a utf8 encoded string. - let oldPos = output.length + const oldPos = output.length output = output + data // The end marker may cross the end of a chunk, so we optimise the search // using the regex lastIndex property.