From 1bb12a5035e5b08e191b1084821377beea5e4d06 Mon Sep 17 00:00:00 2001 From: Brian Gough Date: Wed, 11 Dec 2019 14:43:59 +0000 Subject: [PATCH 01/16] allow pending updates to clear in acceptance tests --- .../coffee/helpers/DocUpdaterClient.coffee | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/services/document-updater/test/acceptance/coffee/helpers/DocUpdaterClient.coffee b/services/document-updater/test/acceptance/coffee/helpers/DocUpdaterClient.coffee index 17067b5bf4..b78f2aa7dd 100644 --- a/services/document-updater/test/acceptance/coffee/helpers/DocUpdaterClient.coffee +++ b/services/document-updater/test/acceptance/coffee/helpers/DocUpdaterClient.coffee @@ -33,7 +33,17 @@ module.exports = DocUpdaterClient = do (update) -> jobs.push (callback) -> DocUpdaterClient.sendUpdate project_id, doc_id, update, callback - async.series jobs, callback + async.series jobs, (err) -> + DocUpdaterClient.waitForPendingUpdates project_id, doc_id, callback + + waitForPendingUpdates: (project_id, doc_id, callback) -> + async.retry {times: 30, interval: 100}, (cb) -> + rclient.llen keys.pendingUpdates({doc_id}), (err, length) -> + if length > 0 + cb(new Error("updates still pending")) + else + cb() + , callback getDoc: (project_id, doc_id, callback = (error, res, body) ->) -> request.get "http://localhost:3003/project/#{project_id}/doc/#{doc_id}", (error, res, body) -> From 3caa0e7c058639d434068e5c88174ada26adfc35 Mon Sep 17 00:00:00 2001 From: Brian Gough Date: Tue, 14 Jan 2020 13:53:50 +0000 Subject: [PATCH 02/16] add failure/retry metrics for web-api requests --- .../app/coffee/PersistenceManager.coffee | 17 ++++++++++ .../PersistenceManagerTests.coffee | 32 +++++++++++++++++-- 2 files changed, 47 insertions(+), 2 deletions(-) diff --git a/services/document-updater/app/coffee/PersistenceManager.coffee b/services/document-updater/app/coffee/PersistenceManager.coffee index 543b9f0c22..f27c5f1b7a 100644 --- a/services/document-updater/app/coffee/PersistenceManager.coffee +++ b/services/document-updater/app/coffee/PersistenceManager.coffee @@ -12,6 +12,21 @@ request = (require("requestretry")).defaults({ # hold us up, and need to bail out quickly if there is a problem. MAX_HTTP_REQUEST_LENGTH = 5000 # 5 seconds +updateMetric = (method, error, response) -> + # find the status, with special handling for connection timeouts + # https://github.com/request/request#timeouts + status = if error?.connect is true + "#{error.code} (connect)" + else if error? + error.code + else if response? + response.statusCode + Metrics.inc method, {status: status} + if error?.attempts > 0 + Metrics.inc "#{method}-attempts", {status: 'error'} + if response?.attempts > 0 + Metrics.inc "#{method}-attempts", {status: 'success'} + module.exports = PersistenceManager = getDoc: (project_id, doc_id, _callback = (error, lines, version, ranges, pathname, projectHistoryId, projectHistoryType) ->) -> timer = new Metrics.Timer("persistenceManager.getDoc") @@ -32,6 +47,7 @@ module.exports = PersistenceManager = jar: false timeout: MAX_HTTP_REQUEST_LENGTH }, (error, res, body) -> + updateMetric('getDoc', error, res) return callback(error) if error? if res.statusCode >= 200 and res.statusCode < 300 try @@ -73,6 +89,7 @@ module.exports = PersistenceManager = jar: false timeout: MAX_HTTP_REQUEST_LENGTH }, (error, res, body) -> + updateMetric('setDoc', error, res) return callback(error) if error? if res.statusCode >= 200 and res.statusCode < 300 return callback null diff --git a/services/document-updater/test/unit/coffee/PersistenceManager/PersistenceManagerTests.coffee b/services/document-updater/test/unit/coffee/PersistenceManager/PersistenceManagerTests.coffee index d1308ad899..a8e2ed9b23 100644 --- a/services/document-updater/test/unit/coffee/PersistenceManager/PersistenceManagerTests.coffee +++ b/services/document-updater/test/unit/coffee/PersistenceManager/PersistenceManagerTests.coffee @@ -15,6 +15,7 @@ describe "PersistenceManager", -> "./Metrics": @Metrics = Timer: class Timer done: sinon.stub() + inc: sinon.stub() "logger-sharelatex": @logger = {log: sinon.stub(), err: sinon.stub()} @project_id = "project-id-123" @projectHistoryId = "history-id-123" @@ -71,9 +72,14 @@ describe "PersistenceManager", -> it "should time the execution", -> @Metrics.Timer::done.called.should.equal true + it "should increment the metric", -> + @Metrics.inc.calledWith("getDoc", {status: 200}).should.equal true + describe "when request returns an error", -> beforeEach -> - @request.callsArgWith(1, @error = new Error("oops"), null, null) + @error = new Error("oops") + @error.code = "EOOPS" + @request.callsArgWith(1, @error, null, null) @PersistenceManager.getDoc(@project_id, @doc_id, @callback) it "should return the error", -> @@ -82,6 +88,9 @@ describe "PersistenceManager", -> it "should time the execution", -> @Metrics.Timer::done.called.should.equal true + it "should increment the metric", -> + @Metrics.inc.calledWith("getDoc", {status: "EOOPS"}).should.equal true + describe "when the request returns 404", -> beforeEach -> @request.callsArgWith(1, null, {statusCode: 404}, "") @@ -93,6 +102,9 @@ describe "PersistenceManager", -> it "should time the execution", -> @Metrics.Timer::done.called.should.equal true + it "should increment the metric", -> + @Metrics.inc.calledWith("getDoc", {status: 404}).should.equal true + describe "when the request returns an error status code", -> beforeEach -> @request.callsArgWith(1, null, {statusCode: 500}, "") @@ -104,6 +116,9 @@ describe "PersistenceManager", -> it "should time the execution", -> @Metrics.Timer::done.called.should.equal true + it "should increment the metric", -> + @Metrics.inc.calledWith("getDoc", {status: 500}).should.equal true + describe "when request returns an doc without lines", -> beforeEach -> delete @webResponse.lines @@ -163,9 +178,14 @@ describe "PersistenceManager", -> it "should time the execution", -> @Metrics.Timer::done.called.should.equal true + it "should increment the metric", -> + @Metrics.inc.calledWith("setDoc", {status: 200}).should.equal true + describe "when request returns an error", -> beforeEach -> - @request.callsArgWith(1, @error = new Error("oops"), null, null) + @error = new Error("oops") + @error.code = "EOOPS" + @request.callsArgWith(1, @error, null, null) @PersistenceManager.setDoc(@project_id, @doc_id, @lines, @version, @ranges, @lastUpdatedAt, @lastUpdatedBy, @callback) it "should return the error", -> @@ -174,6 +194,9 @@ describe "PersistenceManager", -> it "should time the execution", -> @Metrics.Timer::done.called.should.equal true + it "should increment the metric", -> + @Metrics.inc.calledWith("setDoc", {status: "EOOPS"}).should.equal true + describe "when the request returns 404", -> beforeEach -> @request.callsArgWith(1, null, {statusCode: 404}, "") @@ -185,6 +208,9 @@ describe "PersistenceManager", -> it "should time the execution", -> @Metrics.Timer::done.called.should.equal true + it "should increment the metric", -> + @Metrics.inc.calledWith("setDoc", {status: 404}).should.equal true + describe "when the request returns an error status code", -> beforeEach -> @request.callsArgWith(1, null, {statusCode: 500}, "") @@ -196,3 +222,5 @@ describe "PersistenceManager", -> it "should time the execution", -> @Metrics.Timer::done.called.should.equal true + it "should increment the metric", -> + @Metrics.inc.calledWith("setDoc", {status: 500}).should.equal true From 7036803acfd551eb4d074b65a5244069e33ffb67 Mon Sep 17 00:00:00 2001 From: Brian Gough Date: Tue, 14 Jan 2020 15:00:21 +0000 Subject: [PATCH 03/16] add missing argument to metrics.inc also track retries rather than attempts (which is always 1 for a successful request) --- .../app/coffee/PersistenceManager.coffee | 10 +++++----- .../PersistenceManagerTests.coffee | 16 ++++++++-------- 2 files changed, 13 insertions(+), 13 deletions(-) diff --git a/services/document-updater/app/coffee/PersistenceManager.coffee b/services/document-updater/app/coffee/PersistenceManager.coffee index f27c5f1b7a..88b44fd1de 100644 --- a/services/document-updater/app/coffee/PersistenceManager.coffee +++ b/services/document-updater/app/coffee/PersistenceManager.coffee @@ -21,11 +21,11 @@ updateMetric = (method, error, response) -> error.code else if response? response.statusCode - Metrics.inc method, {status: status} - if error?.attempts > 0 - Metrics.inc "#{method}-attempts", {status: 'error'} - if response?.attempts > 0 - Metrics.inc "#{method}-attempts", {status: 'success'} + Metrics.inc method, 1, {status: status} + if error?.attempts > 1 + Metrics.inc "#{method}-retries", 1, {status: 'error'} + if response?.attempts > 1 + Metrics.inc "#{method}-retries", 1, {status: 'success'} module.exports = PersistenceManager = getDoc: (project_id, doc_id, _callback = (error, lines, version, ranges, pathname, projectHistoryId, projectHistoryType) ->) -> diff --git a/services/document-updater/test/unit/coffee/PersistenceManager/PersistenceManagerTests.coffee b/services/document-updater/test/unit/coffee/PersistenceManager/PersistenceManagerTests.coffee index a8e2ed9b23..0ad69c3885 100644 --- a/services/document-updater/test/unit/coffee/PersistenceManager/PersistenceManagerTests.coffee +++ b/services/document-updater/test/unit/coffee/PersistenceManager/PersistenceManagerTests.coffee @@ -73,7 +73,7 @@ describe "PersistenceManager", -> @Metrics.Timer::done.called.should.equal true it "should increment the metric", -> - @Metrics.inc.calledWith("getDoc", {status: 200}).should.equal true + @Metrics.inc.calledWith("getDoc", 1, {status: 200}).should.equal true describe "when request returns an error", -> beforeEach -> @@ -89,7 +89,7 @@ describe "PersistenceManager", -> @Metrics.Timer::done.called.should.equal true it "should increment the metric", -> - @Metrics.inc.calledWith("getDoc", {status: "EOOPS"}).should.equal true + @Metrics.inc.calledWith("getDoc", 1, {status: "EOOPS"}).should.equal true describe "when the request returns 404", -> beforeEach -> @@ -103,7 +103,7 @@ describe "PersistenceManager", -> @Metrics.Timer::done.called.should.equal true it "should increment the metric", -> - @Metrics.inc.calledWith("getDoc", {status: 404}).should.equal true + @Metrics.inc.calledWith("getDoc", 1, {status: 404}).should.equal true describe "when the request returns an error status code", -> beforeEach -> @@ -117,7 +117,7 @@ describe "PersistenceManager", -> @Metrics.Timer::done.called.should.equal true it "should increment the metric", -> - @Metrics.inc.calledWith("getDoc", {status: 500}).should.equal true + @Metrics.inc.calledWith("getDoc", 1, {status: 500}).should.equal true describe "when request returns an doc without lines", -> beforeEach -> @@ -179,7 +179,7 @@ describe "PersistenceManager", -> @Metrics.Timer::done.called.should.equal true it "should increment the metric", -> - @Metrics.inc.calledWith("setDoc", {status: 200}).should.equal true + @Metrics.inc.calledWith("setDoc", 1, {status: 200}).should.equal true describe "when request returns an error", -> beforeEach -> @@ -195,7 +195,7 @@ describe "PersistenceManager", -> @Metrics.Timer::done.called.should.equal true it "should increment the metric", -> - @Metrics.inc.calledWith("setDoc", {status: "EOOPS"}).should.equal true + @Metrics.inc.calledWith("setDoc", 1, {status: "EOOPS"}).should.equal true describe "when the request returns 404", -> beforeEach -> @@ -209,7 +209,7 @@ describe "PersistenceManager", -> @Metrics.Timer::done.called.should.equal true it "should increment the metric", -> - @Metrics.inc.calledWith("setDoc", {status: 404}).should.equal true + @Metrics.inc.calledWith("setDoc", 1, {status: 404}).should.equal true describe "when the request returns an error status code", -> beforeEach -> @@ -223,4 +223,4 @@ describe "PersistenceManager", -> @Metrics.Timer::done.called.should.equal true it "should increment the metric", -> - @Metrics.inc.calledWith("setDoc", {status: 500}).should.equal true + @Metrics.inc.calledWith("setDoc", 1, {status: 500}).should.equal true From fcfa3ecc9d04d06c3fc46eeec60ee78c374137d0 Mon Sep 17 00:00:00 2001 From: Jakob Ackermann Date: Thu, 23 Jan 2020 10:05:57 +0100 Subject: [PATCH 04/16] [misc] upgrade node to 10.18.1 --- services/document-updater/.nvmrc | 2 +- services/document-updater/Dockerfile | 9 +++++-- services/document-updater/Jenkinsfile | 1 + services/document-updater/Makefile | 4 ++- services/document-updater/buildscript.txt | 8 +++--- .../document-updater/docker-compose.ci.yml | 14 +++++----- services/document-updater/docker-compose.yml | 27 +++++++------------ 7 files changed, 32 insertions(+), 33 deletions(-) diff --git a/services/document-updater/.nvmrc b/services/document-updater/.nvmrc index e1e5d1369a..fd26bfb7c5 100644 --- a/services/document-updater/.nvmrc +++ b/services/document-updater/.nvmrc @@ -1 +1 @@ -6.9.5 +10.18.1 diff --git a/services/document-updater/Dockerfile b/services/document-updater/Dockerfile index 59f5e61889..6fe5611b99 100644 --- a/services/document-updater/Dockerfile +++ b/services/document-updater/Dockerfile @@ -1,4 +1,9 @@ -FROM node:6.9.5 as app +# This file was auto-generated, do not edit it directly. +# Instead run bin/update_build_scripts from +# https://github.com/sharelatex/sharelatex-dev-environment +# Version: 1.3.2 + +FROM node:10.18.1 as app WORKDIR /app @@ -12,7 +17,7 @@ COPY . /app RUN npm run compile:all -FROM node:6.9.5 +FROM node:10.18.1 COPY --from=app /app /app diff --git a/services/document-updater/Jenkinsfile b/services/document-updater/Jenkinsfile index 2862de8f47..92db215930 100644 --- a/services/document-updater/Jenkinsfile +++ b/services/document-updater/Jenkinsfile @@ -16,6 +16,7 @@ pipeline { } stages { + stage('Install') { steps { withCredentials([usernamePassword(credentialsId: 'GITHUB_INTEGRATION', usernameVariable: 'GH_AUTH_USERNAME', passwordVariable: 'GH_AUTH_PASSWORD')]) { diff --git a/services/document-updater/Makefile b/services/document-updater/Makefile index 73f63edba8..b7085c7631 100644 --- a/services/document-updater/Makefile +++ b/services/document-updater/Makefile @@ -1,7 +1,7 @@ # This file was auto-generated, do not edit it directly. # Instead run bin/update_build_scripts from # https://github.com/sharelatex/sharelatex-dev-environment -# Version: 1.1.24 +# Version: 1.3.2 BUILD_NUMBER ?= local BRANCH_NAME ?= $(shell git rev-parse --abbrev-ref HEAD) @@ -36,6 +36,7 @@ test_clean: test_acceptance_pre_run: @[ ! -f test/acceptance/js/scripts/pre-run ] && echo "document-updater has no pre acceptance tests task" || $(DOCKER_COMPOSE) run --rm test_acceptance test/acceptance/js/scripts/pre-run + build: docker build --pull --tag ci/$(PROJECT_NAME):$(BRANCH_NAME)-$(BUILD_NUMBER) \ --tag gcr.io/overleaf-ops/$(PROJECT_NAME):$(BRANCH_NAME)-$(BUILD_NUMBER) \ @@ -48,4 +49,5 @@ publish: docker push $(DOCKER_REPO)/$(PROJECT_NAME):$(BRANCH_NAME)-$(BUILD_NUMBER) + .PHONY: clean test test_unit test_acceptance test_clean build publish diff --git a/services/document-updater/buildscript.txt b/services/document-updater/buildscript.txt index a9a1b603d3..2c57ac832a 100644 --- a/services/document-updater/buildscript.txt +++ b/services/document-updater/buildscript.txt @@ -1,10 +1,10 @@ document-updater +--public-repo=True --language=coffeescript ---node-version=6.9.5 +--env-add= +--node-version=10.18.1 --acceptance-creds=None --dependencies=mongo,redis --docker-repos=gcr.io/overleaf-ops ---build-target=docker ---script-version=1.1.24 --env-pass-through= ---public-repo=True +--script-version=1.3.2 diff --git a/services/document-updater/docker-compose.ci.yml b/services/document-updater/docker-compose.ci.yml index c78d90e8ed..59bee48106 100644 --- a/services/document-updater/docker-compose.ci.yml +++ b/services/document-updater/docker-compose.ci.yml @@ -1,9 +1,9 @@ # This file was auto-generated, do not edit it directly. # Instead run bin/update_build_scripts from # https://github.com/sharelatex/sharelatex-dev-environment -# Version: 1.1.24 +# Version: 1.3.2 -version: "2" +version: "2.1" services: test_unit: @@ -25,13 +25,14 @@ services: MOCHA_GREP: ${MOCHA_GREP} NODE_ENV: test depends_on: - - mongo - - redis + mongo: + condition: service_healthy + redis: + condition: service_healthy user: node command: npm run test:acceptance:_run - tar: build: . image: ci/$PROJECT_NAME:$BRANCH_NAME-$BUILD_NUMBER @@ -39,9 +40,8 @@ services: - ./:/tmp/build/ command: tar -czf /tmp/build/build.tar.gz --exclude=build.tar.gz --exclude-vcs . user: root - redis: image: redis mongo: - image: mongo:3.4 + image: mongo:3.6 diff --git a/services/document-updater/docker-compose.yml b/services/document-updater/docker-compose.yml index 6dc90009ca..2b0581294d 100644 --- a/services/document-updater/docker-compose.yml +++ b/services/document-updater/docker-compose.yml @@ -1,13 +1,13 @@ # This file was auto-generated, do not edit it directly. # Instead run bin/update_build_scripts from # https://github.com/sharelatex/sharelatex-dev-environment -# Version: 1.1.24 +# Version: 1.3.2 -version: "2" +version: "2.1" services: test_unit: - image: node:6.9.5 + image: node:10.18.1 volumes: - .:/app working_dir: /app @@ -18,7 +18,7 @@ services: user: node test_acceptance: - build: . + image: node:10.18.1 volumes: - .:/app working_dir: /app @@ -32,24 +32,15 @@ services: NODE_ENV: test user: node depends_on: - - mongo - - redis + mongo: + condition: service_healthy + redis: + condition: service_healthy command: npm run test:acceptance - - - tar: - build: . - image: ci/$PROJECT_NAME:$BRANCH_NAME-$BUILD_NUMBER - volumes: - - ./:/tmp/build/ - command: tar -czf /tmp/build/build.tar.gz --exclude=build.tar.gz --exclude-vcs . - user: root - redis: image: redis mongo: - image: mongo:3.4 - + image: mongo:3.6 From 338d3609f5a5dbf98bc706c14fbbd9b3a4a36558 Mon Sep 17 00:00:00 2001 From: Brian Gough Date: Thu, 30 Jan 2020 15:17:13 +0000 Subject: [PATCH 05/16] add comment about null byte check --- services/document-updater/app/coffee/RedisManager.coffee | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/services/document-updater/app/coffee/RedisManager.coffee b/services/document-updater/app/coffee/RedisManager.coffee index 9e2edbd99d..ca4151d299 100644 --- a/services/document-updater/app/coffee/RedisManager.coffee +++ b/services/document-updater/app/coffee/RedisManager.coffee @@ -36,6 +36,8 @@ module.exports = RedisManager = docLines = JSON.stringify(docLines) if docLines.indexOf("\u0000") != -1 error = new Error("null bytes found in doc lines") + # this check was added to catch memory corruption in JSON.stringify. + # It sometimes returned null bytes at the end of the string. logger.error {err: error, doc_id: doc_id, docLines: docLines}, error.message return callback(error) docHash = RedisManager._computeHash(docLines) @@ -224,12 +226,14 @@ module.exports = RedisManager = for op in jsonOps if op.indexOf("\u0000") != -1 error = new Error("null bytes found in jsonOps") + # this check was added to catch memory corruption in JSON.stringify logger.error {err: error, doc_id: doc_id, jsonOps: jsonOps}, error.message return callback(error) newDocLines = JSON.stringify(docLines) if newDocLines.indexOf("\u0000") != -1 error = new Error("null bytes found in doc lines") + # this check was added to catch memory corruption in JSON.stringify logger.error {err: error, doc_id: doc_id, newDocLines: newDocLines}, error.message return callback(error) newHash = RedisManager._computeHash(newDocLines) @@ -243,6 +247,7 @@ module.exports = RedisManager = return callback(error) if ranges? and ranges.indexOf("\u0000") != -1 error = new Error("null bytes found in ranges") + # this check was added to catch memory corruption in JSON.stringify logger.error err: error, doc_id: doc_id, ranges: ranges, error.message return callback(error) multi = rclient.multi() From 10d177f41bc2d07e311e3d30c1e108466b67bf86 Mon Sep 17 00:00:00 2001 From: Jakob Ackermann Date: Fri, 7 Feb 2020 12:24:57 +0100 Subject: [PATCH 06/16] [misc] test/unit: stub the correct redis settings --- .../unit/coffee/DispatchManager/DispatchManagerTests.coffee | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/services/document-updater/test/unit/coffee/DispatchManager/DispatchManagerTests.coffee b/services/document-updater/test/unit/coffee/DispatchManager/DispatchManagerTests.coffee index af36d10a31..773a85afd1 100644 --- a/services/document-updater/test/unit/coffee/DispatchManager/DispatchManagerTests.coffee +++ b/services/document-updater/test/unit/coffee/DispatchManager/DispatchManagerTests.coffee @@ -13,7 +13,7 @@ describe "DispatchManager", -> "logger-sharelatex": @logger = { log: sinon.stub(), error: sinon.stub(), warn: sinon.stub() } "settings-sharelatex": @settings = redis: - realtime: {} + documentupdater: {} "redis-sharelatex": @redis = {} "./RateLimitManager": {} "./Errors": Errors From 4070b0ef6f0ad787644b1d9ce22373c89b47e4ba Mon Sep 17 00:00:00 2001 From: Jakob Ackermann Date: Thu, 16 May 2019 14:58:32 +0200 Subject: [PATCH 07/16] [misc] tests: fix a static test `ProjectHistoryRedisManager.queueOps` is masked and can not push changes into redis during the unittest. Signed-off-by: Jakob Ackermann --- .../test/unit/coffee/RedisManager/RedisManagerTests.coffee | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/services/document-updater/test/unit/coffee/RedisManager/RedisManagerTests.coffee b/services/document-updater/test/unit/coffee/RedisManager/RedisManagerTests.coffee index 99035a32b3..b666163762 100644 --- a/services/document-updater/test/unit/coffee/RedisManager/RedisManagerTests.coffee +++ b/services/document-updater/test/unit/coffee/RedisManager/RedisManagerTests.coffee @@ -445,13 +445,12 @@ describe "RedisManager", -> describe "with project history disabled", -> beforeEach -> - @rclient.rpush = sinon.stub() @settings.apis.project_history.enabled = false @RedisManager.getDocVersion.withArgs(@doc_id).yields(null, @version - @ops.length) @RedisManager.updateDocument @project_id, @doc_id, @lines, @version, @ops, @ranges, @updateMeta, @callback it "should not push the updates into the project history ops list", -> - @rclient.rpush.called.should.equal false + @ProjectHistoryRedisManager.queueOps.called.should.equal false it "should call the callback", -> @callback @@ -493,7 +492,6 @@ describe "RedisManager", -> describe "with no updates", -> beforeEach -> - @rclient.rpush = sinon.stub().callsArgWith(1, null, @project_update_list_length) @RedisManager.getDocVersion.withArgs(@doc_id).yields(null, @version) @RedisManager.updateDocument @project_id, @doc_id, @lines, @version, [], @ranges, @updateMeta, @callback @@ -503,7 +501,7 @@ describe "RedisManager", -> .should.equal false it "should not try to enqueue project updates", -> - @rclient.rpush + @ProjectHistoryRedisManager.queueOps .called .should.equal false From afe43fa252c431fcaf281c63c77b3bd02eca0f93 Mon Sep 17 00:00:00 2001 From: Brian Gough Date: Fri, 7 Feb 2020 14:16:31 +0000 Subject: [PATCH 08/16] update to node 10.19.0 --- services/document-updater/.nvmrc | 2 +- services/document-updater/Dockerfile | 4 ++-- services/document-updater/buildscript.txt | 2 +- services/document-updater/docker-compose.yml | 4 ++-- 4 files changed, 6 insertions(+), 6 deletions(-) diff --git a/services/document-updater/.nvmrc b/services/document-updater/.nvmrc index fd26bfb7c5..5b7269c0a9 100644 --- a/services/document-updater/.nvmrc +++ b/services/document-updater/.nvmrc @@ -1 +1 @@ -10.18.1 +10.19.0 diff --git a/services/document-updater/Dockerfile b/services/document-updater/Dockerfile index 6fe5611b99..1ff1ecd5c5 100644 --- a/services/document-updater/Dockerfile +++ b/services/document-updater/Dockerfile @@ -3,7 +3,7 @@ # https://github.com/sharelatex/sharelatex-dev-environment # Version: 1.3.2 -FROM node:10.18.1 as app +FROM node:10.19.0 as app WORKDIR /app @@ -17,7 +17,7 @@ COPY . /app RUN npm run compile:all -FROM node:10.18.1 +FROM node:10.19.0 COPY --from=app /app /app diff --git a/services/document-updater/buildscript.txt b/services/document-updater/buildscript.txt index 2c57ac832a..094b13d978 100644 --- a/services/document-updater/buildscript.txt +++ b/services/document-updater/buildscript.txt @@ -2,7 +2,7 @@ document-updater --public-repo=True --language=coffeescript --env-add= ---node-version=10.18.1 +--node-version=10.19.0 --acceptance-creds=None --dependencies=mongo,redis --docker-repos=gcr.io/overleaf-ops diff --git a/services/document-updater/docker-compose.yml b/services/document-updater/docker-compose.yml index 2b0581294d..d3bc8c7d98 100644 --- a/services/document-updater/docker-compose.yml +++ b/services/document-updater/docker-compose.yml @@ -7,7 +7,7 @@ version: "2.1" services: test_unit: - image: node:10.18.1 + image: node:10.19.0 volumes: - .:/app working_dir: /app @@ -18,7 +18,7 @@ services: user: node test_acceptance: - image: node:10.18.1 + image: node:10.19.0 volumes: - .:/app working_dir: /app From 9cfc59734a4c1e0b1e5e51b08fbec2632d5d0bac Mon Sep 17 00:00:00 2001 From: Jakob Ackermann Date: Mon, 10 Feb 2020 17:10:39 +0100 Subject: [PATCH 09/16] [misc] update the build scripts to 1.3.5 --- services/document-updater/Dockerfile | 10 +++++----- services/document-updater/Makefile | 7 ++++++- services/document-updater/buildscript.txt | 2 +- services/document-updater/docker-compose.ci.yml | 4 ++-- services/document-updater/docker-compose.yml | 4 ++-- 5 files changed, 16 insertions(+), 11 deletions(-) diff --git a/services/document-updater/Dockerfile b/services/document-updater/Dockerfile index 1ff1ecd5c5..e538fb48d9 100644 --- a/services/document-updater/Dockerfile +++ b/services/document-updater/Dockerfile @@ -1,12 +1,14 @@ # This file was auto-generated, do not edit it directly. # Instead run bin/update_build_scripts from # https://github.com/sharelatex/sharelatex-dev-environment -# Version: 1.3.2 +# Version: 1.3.5 -FROM node:10.19.0 as app +FROM node:10.19.0 as base WORKDIR /app +FROM base as app + #wildcard as some files may not be in all repos COPY package*.json npm-shrink*.json /app/ @@ -17,11 +19,9 @@ COPY . /app RUN npm run compile:all -FROM node:10.19.0 +FROM base COPY --from=app /app /app - -WORKDIR /app USER node CMD ["node", "--expose-gc", "app.js"] diff --git a/services/document-updater/Makefile b/services/document-updater/Makefile index b7085c7631..64646d796f 100644 --- a/services/document-updater/Makefile +++ b/services/document-updater/Makefile @@ -1,7 +1,7 @@ # This file was auto-generated, do not edit it directly. # Instead run bin/update_build_scripts from # https://github.com/sharelatex/sharelatex-dev-environment -# Version: 1.3.2 +# Version: 1.3.5 BUILD_NUMBER ?= local BRANCH_NAME ?= $(shell git rev-parse --abbrev-ref HEAD) @@ -28,9 +28,14 @@ test_unit: test_acceptance: test_clean test_acceptance_pre_run test_acceptance_run +test_acceptance_debug: test_clean test_acceptance_pre_run test_acceptance_run_debug + test_acceptance_run: @[ ! -d test/acceptance ] && echo "document-updater has no acceptance tests" || $(DOCKER_COMPOSE) run --rm test_acceptance +test_acceptance_run_debug: + @[ ! -d test/acceptance ] && echo "document-updater has no acceptance tests" || $(DOCKER_COMPOSE) run -p 127.0.0.9:19999:19999 --rm test_acceptance npm run test:acceptance -- --inspect=0.0.0.0:19999 --inspect-brk + test_clean: $(DOCKER_COMPOSE) down -v -t 0 diff --git a/services/document-updater/buildscript.txt b/services/document-updater/buildscript.txt index 094b13d978..b7928a7044 100644 --- a/services/document-updater/buildscript.txt +++ b/services/document-updater/buildscript.txt @@ -7,4 +7,4 @@ document-updater --dependencies=mongo,redis --docker-repos=gcr.io/overleaf-ops --env-pass-through= ---script-version=1.3.2 +--script-version=1.3.5 diff --git a/services/document-updater/docker-compose.ci.yml b/services/document-updater/docker-compose.ci.yml index 59bee48106..b99da9b18e 100644 --- a/services/document-updater/docker-compose.ci.yml +++ b/services/document-updater/docker-compose.ci.yml @@ -1,9 +1,9 @@ # This file was auto-generated, do not edit it directly. # Instead run bin/update_build_scripts from # https://github.com/sharelatex/sharelatex-dev-environment -# Version: 1.3.2 +# Version: 1.3.5 -version: "2.1" +version: "2.3" services: test_unit: diff --git a/services/document-updater/docker-compose.yml b/services/document-updater/docker-compose.yml index d3bc8c7d98..6a1bbb1005 100644 --- a/services/document-updater/docker-compose.yml +++ b/services/document-updater/docker-compose.yml @@ -1,9 +1,9 @@ # This file was auto-generated, do not edit it directly. # Instead run bin/update_build_scripts from # https://github.com/sharelatex/sharelatex-dev-environment -# Version: 1.3.2 +# Version: 1.3.5 -version: "2.1" +version: "2.3" services: test_unit: From 26d8d07e7af28daeabb3f48cab4eb8b830dbe328 Mon Sep 17 00:00:00 2001 From: Brian Gough Date: Tue, 11 Feb 2020 14:32:49 +0000 Subject: [PATCH 10/16] remove unused redis settings from rate limit unit test --- .../test/unit/coffee/RateLimitManager/RateLimitManager.coffee | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/services/document-updater/test/unit/coffee/RateLimitManager/RateLimitManager.coffee b/services/document-updater/test/unit/coffee/RateLimitManager/RateLimitManager.coffee index 866532a4da..fe5dc95327 100644 --- a/services/document-updater/test/unit/coffee/RateLimitManager/RateLimitManager.coffee +++ b/services/document-updater/test/unit/coffee/RateLimitManager/RateLimitManager.coffee @@ -9,9 +9,7 @@ describe "RateLimitManager", -> beforeEach -> @RateLimitManager = SandboxedModule.require modulePath, requires: "logger-sharelatex": @logger = { log: sinon.stub() } - "settings-sharelatex": @settings = - redis: - realtime: {} + "settings-sharelatex": @settings = {} "./Metrics": @Metrics = Timer: class Timer done: sinon.stub() From 81e21c5e76c3f7a4e9c9c15d38585574bce83b22 Mon Sep 17 00:00:00 2001 From: Brian Gough Date: Wed, 12 Feb 2020 12:37:00 +0000 Subject: [PATCH 11/16] remove unused .travis.yml file --- services/document-updater/.travis.yml | 11 ----------- 1 file changed, 11 deletions(-) delete mode 100644 services/document-updater/.travis.yml diff --git a/services/document-updater/.travis.yml b/services/document-updater/.travis.yml deleted file mode 100644 index febdbb55a3..0000000000 --- a/services/document-updater/.travis.yml +++ /dev/null @@ -1,11 +0,0 @@ -language: node_js - -before_install: - - npm install -g grunt-cli - -install: - - npm install - - grunt install - -script: - - grunt test:unit From af4211d167f2e9d8c6950e0af6a07a0271c863aa Mon Sep 17 00:00:00 2001 From: Jakob Ackermann Date: Wed, 12 Feb 2020 14:39:51 +0100 Subject: [PATCH 12/16] [misc] rename npm-shrinkwrap.json to package-lock.json and run npm i --- .../document-updater/{npm-shrinkwrap.json => package-lock.json} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename services/document-updater/{npm-shrinkwrap.json => package-lock.json} (100%) diff --git a/services/document-updater/npm-shrinkwrap.json b/services/document-updater/package-lock.json similarity index 100% rename from services/document-updater/npm-shrinkwrap.json rename to services/document-updater/package-lock.json From c4bec0dcc7028a59a04a42543c551261db1b6315 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Fri, 14 Feb 2020 09:56:23 +0000 Subject: [PATCH 13/16] Bump extend from 3.0.1 to 3.0.2 Bumps [extend](https://github.com/justmoon/node-extend) from 3.0.1 to 3.0.2. - [Release notes](https://github.com/justmoon/node-extend/releases) - [Changelog](https://github.com/justmoon/node-extend/blob/master/CHANGELOG.md) - [Commits](https://github.com/justmoon/node-extend/compare/v3.0.1...v3.0.2) Signed-off-by: dependabot[bot] --- services/document-updater/package-lock.json | 25 +++------------------ 1 file changed, 3 insertions(+), 22 deletions(-) diff --git a/services/document-updater/package-lock.json b/services/document-updater/package-lock.json index 964ffbc4d2..28faa73192 100644 --- a/services/document-updater/package-lock.json +++ b/services/document-updater/package-lock.json @@ -20,13 +20,6 @@ "pify": "^4.0.1", "retry-request": "^4.0.0", "teeny-request": "^3.11.3" - }, - "dependencies": { - "extend": { - "version": "3.0.2", - "resolved": "https://registry.npmjs.org/extend/-/extend-3.0.2.tgz", - "integrity": "sha512-fjquC59cD7CyW6urNXK0FBufkZcoiGG80wTuPujX590cB5Ttln20E2UB4S/WARVqhXffZl2LNgS+gQdPIIim/g==" - } } }, "@google-cloud/debug-agent": { @@ -1088,9 +1081,9 @@ } }, "extend": { - "version": "3.0.1", - "resolved": "https://registry.npmjs.org/extend/-/extend-3.0.1.tgz", - "integrity": "sha1-p1Xqe8Gt/MWjHOfnYtuq3F5jZEQ=" + "version": "3.0.2", + "resolved": "https://registry.npmjs.org/extend/-/extend-3.0.2.tgz", + "integrity": "sha512-fjquC59cD7CyW6urNXK0FBufkZcoiGG80wTuPujX590cB5Ttln20E2UB4S/WARVqhXffZl2LNgS+gQdPIIim/g==" }, "extsprintf": { "version": "1.3.0", @@ -1204,13 +1197,6 @@ "extend": "^3.0.2", "https-proxy-agent": "^2.2.1", "node-fetch": "^2.3.0" - }, - "dependencies": { - "extend": { - "version": "3.0.2", - "resolved": "https://registry.npmjs.org/extend/-/extend-3.0.2.tgz", - "integrity": "sha512-fjquC59cD7CyW6urNXK0FBufkZcoiGG80wTuPujX590cB5Ttln20E2UB4S/WARVqhXffZl2LNgS+gQdPIIim/g==" - } } }, "gcp-metadata": { @@ -1594,11 +1580,6 @@ "nan": "^2.10.0" } }, - "extend": { - "version": "3.0.2", - "resolved": "https://registry.npmjs.org/extend/-/extend-3.0.2.tgz", - "integrity": "sha512-fjquC59cD7CyW6urNXK0FBufkZcoiGG80wTuPujX590cB5Ttln20E2UB4S/WARVqhXffZl2LNgS+gQdPIIim/g==" - }, "fast-deep-equal": { "version": "2.0.1", "resolved": "https://registry.npmjs.org/fast-deep-equal/-/fast-deep-equal-2.0.1.tgz", From 2acd4c0f23b39439ae026a8a5e04780cb8b67c9f Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Fri, 14 Feb 2020 09:56:24 +0000 Subject: [PATCH 14/16] Bump stringstream from 0.0.5 to 0.0.6 Bumps [stringstream](https://github.com/mhart/StringStream) from 0.0.5 to 0.0.6. - [Release notes](https://github.com/mhart/StringStream/releases) - [Commits](https://github.com/mhart/StringStream/compare/v0.0.5...v0.0.6) Signed-off-by: dependabot[bot] --- services/document-updater/package-lock.json | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/services/document-updater/package-lock.json b/services/document-updater/package-lock.json index 964ffbc4d2..3be60492b6 100644 --- a/services/document-updater/package-lock.json +++ b/services/document-updater/package-lock.json @@ -2874,9 +2874,9 @@ } }, "stringstream": { - "version": "0.0.5", - "resolved": "https://registry.npmjs.org/stringstream/-/stringstream-0.0.5.tgz", - "integrity": "sha1-TkhM1N5aC7vuGORjB3EKioFiGHg=" + "version": "0.0.6", + "resolved": "https://registry.npmjs.org/stringstream/-/stringstream-0.0.6.tgz", + "integrity": "sha512-87GEBAkegbBcweToUrdzf3eLhWNg06FJTebl4BVJz/JgWy8CvEr9dRtX5qWphiynMSQlxxi+QqN0z5T32SLlhA==" }, "tdigest": { "version": "0.1.1", From 3fb0ed414cf0282905c791d52475c9a1acabeded Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Fri, 14 Feb 2020 09:56:25 +0000 Subject: [PATCH 15/16] Bump sshpk from 1.13.1 to 1.16.1 Bumps [sshpk](https://github.com/joyent/node-sshpk) from 1.13.1 to 1.16.1. - [Release notes](https://github.com/joyent/node-sshpk/releases) - [Commits](https://github.com/joyent/node-sshpk/compare/v1.13.1...v1.16.1) Signed-off-by: dependabot[bot] --- services/document-updater/package-lock.json | 29 ++++++++++++--------- 1 file changed, 17 insertions(+), 12 deletions(-) diff --git a/services/document-updater/package-lock.json b/services/document-updater/package-lock.json index 964ffbc4d2..fdc13342e3 100644 --- a/services/document-updater/package-lock.json +++ b/services/document-updater/package-lock.json @@ -453,7 +453,6 @@ "version": "1.0.1", "resolved": "https://registry.npmjs.org/bcrypt-pbkdf/-/bcrypt-pbkdf-1.0.1.tgz", "integrity": "sha1-Y7xdy2EzG5K8Bf1SiVPDNGKgb40=", - "optional": true, "requires": { "tweetnacl": "^0.14.3" } @@ -934,7 +933,6 @@ "version": "0.1.1", "resolved": "https://registry.npmjs.org/ecc-jsbn/-/ecc-jsbn-0.1.1.tgz", "integrity": "sha1-D8c6ntXw1Tw4GTOYUj735UN3dQU=", - "optional": true, "requires": { "jsbn": "~0.1.0" } @@ -1442,8 +1440,7 @@ "jsbn": { "version": "0.1.1", "resolved": "https://registry.npmjs.org/jsbn/-/jsbn-0.1.1.tgz", - "integrity": "sha1-peZUwuWi3rXyAdls77yoDA7y9RM=", - "optional": true + "integrity": "sha1-peZUwuWi3rXyAdls77yoDA7y9RM=" }, "json-bigint": { "version": "0.3.0", @@ -2657,6 +2654,11 @@ "integrity": "sha512-gH8eh2nZudPQO6TytOvbxnuhYBOvDBBLW52tz5q6X58lJcd/tkmqFR+5Z9adS8aJtURSXWThWy/xJtJwixErvg==", "optional": true }, + "safer-buffer": { + "version": "2.1.2", + "resolved": "https://registry.npmjs.org/safer-buffer/-/safer-buffer-2.1.2.tgz", + "integrity": "sha512-YZo3K82SD7Riyi0E1EQPojLz7kpepnSQI9IyPbHHg1XXXevb5dJI7tpyN2ADxGcQbHG7vcyRHk0cbwqcQriUtg==" + }, "sandboxed-module": { "version": "0.2.2", "resolved": "https://registry.npmjs.org/sandboxed-module/-/sandboxed-module-0.2.2.tgz", @@ -2788,9 +2790,9 @@ } }, "sshpk": { - "version": "1.13.1", - "resolved": "https://registry.npmjs.org/sshpk/-/sshpk-1.13.1.tgz", - "integrity": "sha1-US322mKHFEMW3EwY/hzx2UBzm+M=", + "version": "1.16.1", + "resolved": "https://registry.npmjs.org/sshpk/-/sshpk-1.16.1.tgz", + "integrity": "sha512-HXXqVUq7+pcKeLqqZj6mHFUMvXtOJt1uoUx09pFW6011inTMxqI8BA8PM95myrIyyKwdnzjdFjLiE6KBPVtJIg==", "requires": { "asn1": "~0.2.3", "assert-plus": "^1.0.0", @@ -2799,13 +2801,17 @@ "ecc-jsbn": "~0.1.1", "getpass": "^0.1.1", "jsbn": "~0.1.0", + "safer-buffer": "^2.0.2", "tweetnacl": "~0.14.0" }, "dependencies": { "asn1": { - "version": "0.2.3", - "resolved": "https://registry.npmjs.org/asn1/-/asn1-0.2.3.tgz", - "integrity": "sha1-2sh4dxPJlmhJ/IGAd36+nB3fO4Y=" + "version": "0.2.4", + "resolved": "https://registry.npmjs.org/asn1/-/asn1-0.2.4.tgz", + "integrity": "sha512-jxwzQpLQjSmWXgwaCZE9Nz+glAG01yF1QnWgbhGwHI5A6FRIEY6IVqtHhIepHqI7/kyEyQEagBC5mBEFlIYvdg==", + "requires": { + "safer-buffer": "~2.1.0" + } }, "assert-plus": { "version": "1.0.0", @@ -2949,8 +2955,7 @@ "tweetnacl": { "version": "0.14.5", "resolved": "https://registry.npmjs.org/tweetnacl/-/tweetnacl-0.14.5.tgz", - "integrity": "sha1-WuaBd/GS1EViadEIr6k/+HQ/T2Q=", - "optional": true + "integrity": "sha1-WuaBd/GS1EViadEIr6k/+HQ/T2Q=" }, "type-is": { "version": "1.3.1", From 9a92cd7b31b47cc5c036bf3b19e35ba72e0972ef Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Fri, 14 Feb 2020 09:56:26 +0000 Subject: [PATCH 16/16] Bump lodash from 4.17.4 to 4.17.13 Bumps [lodash](https://github.com/lodash/lodash) from 4.17.4 to 4.17.13. - [Release notes](https://github.com/lodash/lodash/releases) - [Commits](https://github.com/lodash/lodash/compare/4.17.4...4.17.13) Signed-off-by: dependabot[bot] --- services/document-updater/package-lock.json | 6 +++--- services/document-updater/package.json | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/services/document-updater/package-lock.json b/services/document-updater/package-lock.json index 964ffbc4d2..d1a9986887 100644 --- a/services/document-updater/package-lock.json +++ b/services/document-updater/package-lock.json @@ -1511,9 +1511,9 @@ "integrity": "sha1-SjGI1CkbZrT2XtuZ+AaqmuKTWSo=" }, "lodash": { - "version": "4.17.4", - "resolved": "https://registry.npmjs.org/lodash/-/lodash-4.17.4.tgz", - "integrity": "sha1-eCA6TRwyiuHYbcpkYONptX9AVa4=" + "version": "4.17.13", + "resolved": "https://registry.npmjs.org/lodash/-/lodash-4.17.13.tgz", + "integrity": "sha512-vm3/XWXfWtRua0FkUyEHBZy8kCPjErNBT9fJx8Zvs+U6zjqPbTUOpkaoum3O5uiA8sm+yNMHXfYkTUHFoMxFNA==" }, "lodash.defaults": { "version": "4.2.0", diff --git a/services/document-updater/package.json b/services/document-updater/package.json index f80251248b..eea5c9d88b 100644 --- a/services/document-updater/package.json +++ b/services/document-updater/package.json @@ -23,7 +23,7 @@ "async": "^2.5.0", "coffee-script": "~1.7.0", "express": "3.11.0", - "lodash": "^4.17.4", + "lodash": "^4.17.13", "logger-sharelatex": "^1.7.0", "lynx": "0.0.11", "metrics-sharelatex": "^2.2.0",