Merge pull request #39 from sharelatex/csh-ho-docker-issue-1338-bulk-upgrade

Services bulk upgrade - document-updater

services/document-updater/.gitignore

@@ -39,6 +39,7 @@ Thumbs.db
 
 app.js
 app/js/*
+app.map
 
 test/unit/js/*
 test/acceptance/js/*

services/document-updater/Dockerfile

@@ -19,4 +19,4 @@ COPY --from=app /app /app
 WORKDIR /app
 USER node
 
-CMD ["node","app.js"]
+CMD ["node", "--expose-gc", "app.js"]

services/document-updater/Makefile

@@ -1,7 +1,7 @@
 # This file was auto-generated, do not edit it directly.
 # Instead run bin/update_build_scripts from
 # https://github.com/sharelatex/sharelatex-dev-environment
-# Version: 1.1.9
+# Version: 1.1.10
 
 BUILD_NUMBER ?= local
 BRANCH_NAME ?= $(shell git rev-parse --abbrev-ref HEAD)

services/document-updater/app.coffee

@@ -26,6 +26,7 @@ app.configure ->
 	app.use(Metrics.http.monitor(logger));
 	app.use express.bodyParser()
 	app.use app.router
+Metrics.injectMetricsRoute(app)
 
 DispatchManager.createAndStartDispatchers(Settings.dispatcherCount || 10)
 

services/document-updater/app/coffee/HistoryManager.coffee

@@ -20,16 +20,25 @@ module.exports = HistoryManager =
 			else if res.statusCode < 200 and res.statusCode >= 300
 				logger.error { doc_id, project_id }, "track changes api returned a failure status code: #{res.statusCode}"
 
+	# flush changes in the background
 	flushProjectChangesAsync: (project_id) ->
 		return if !Settings.apis?.project_history?.enabled
+		HistoryManager.flushProjectChanges project_id, ->
 
+	# flush changes and callback (for when we need to know the queue is flushed)
+	flushProjectChanges: (project_id, callback = (error) ->) ->
+		return callback() if !Settings.apis?.project_history?.enabled
 		url = "#{Settings.apis.project_history.url}/project/#{project_id}/flush"
 		logger.log { project_id, url }, "flushing doc in project history api"
 		request.post url, (error, res, body)->
 			if error?
 				logger.error { error, project_id}, "project history doc to track changes api"
+				return callback(error)
 			else if res.statusCode < 200 and res.statusCode >= 300
 				logger.error { project_id }, "project history api returned a failure status code: #{res.statusCode}"
+				return callback(error)
+			else
+				return callback()
 
 	FLUSH_DOC_EVERY_N_OPS: 100
 	FLUSH_PROJECT_EVERY_N_OPS: 500

services/document-updater/app/coffee/ProjectManager.coffee

@@ -60,14 +60,17 @@ module.exports = ProjectManager =
 
 			logger.log project_id: project_id, doc_ids: doc_ids, "deleting docs"
 			async.series jobs, () ->
-				# There is no harm in flushing project history if the previous call
-				# failed and sometimes it is required
-				HistoryManager.flushProjectChangesAsync project_id
-
-				if errors.length > 0
-					callback new Error("Errors deleting docs. See log for details")
-				else
-					callback(null)
+				# When deleting the project here we want to ensure that project
+				# history is completely flushed because the project may be
+				# deleted in web after this call completes, and so further
+				# attempts to flush would fail after that.
+				HistoryManager.flushProjectChanges project_id, (error) ->
+					if errors.length > 0
+						callback new Error("Errors deleting docs. See log for details")
+					else if error?
+						callback(error)
+					else
+						callback(null)
 
 	getProjectDocsAndFlushIfOld: (project_id, projectStateHash, excludeVersions = {}, _callback = (error, docs) ->) ->
 		timer = new Metrics.Timer("projectManager.getProjectDocsAndFlushIfOld")

services/document-updater/buildscript.txt

@@ -1,4 +1,4 @@
---script-version=1.1.9
+--script-version=1.1.10
 document-updater
 --node-version=6.9.5
 --acceptance-creds=None

services/document-updater/docker-compose.ci.yml

@@ -1,7 +1,7 @@
 # This file was auto-generated, do not edit it directly.
 # Instead run bin/update_build_scripts from
 # https://github.com/sharelatex/sharelatex-dev-environment
-# Version: 1.1.9
+# Version: 1.1.10
 
 version: "2"
 

services/document-updater/docker-compose.yml

@@ -1,7 +1,7 @@
 # This file was auto-generated, do not edit it directly.
 # Instead run bin/update_build_scripts from
 # https://github.com/sharelatex/sharelatex-dev-environment
-# Version: 1.1.9
+# Version: 1.1.10
 
 version: "2"
 

services/document-updater/package.json

@@ -7,7 +7,7 @@
     "url": "https://github.com/sharelatex/document-updater-sharelatex.git"
   },
   "scripts": {
-    "compile:app": "([ -e app/coffee ] && coffee $COFFEE_OPTIONS -o app/js -c app/coffee || echo 'No CoffeeScript folder to compile') && ( [ -e app.coffee ] && coffee $COFFEE_OPTIONS -c app.coffee || echo 'No CoffeeScript app to compile')",
+    "compile:app": "([ -e app/coffee ] && coffee -m $COFFEE_OPTIONS -o app/js -c app/coffee || echo 'No CoffeeScript folder to compile') && ( [ -e app.coffee ] && coffee -m $COFFEE_OPTIONS -c app.coffee || echo 'No CoffeeScript app to compile')",
     "start": "npm run compile:app && node $NODE_APP_OPTIONS app.js",
     "test:acceptance:_run": "mocha --recursive --reporter spec --timeout 30000 --exit $@ test/acceptance/js",
     "test:acceptance": "npm run compile:app && npm run compile:acceptance_tests && npm run test:acceptance:_run -- --grep=$MOCHA_GREP",
@@ -23,9 +23,9 @@
     "async": "^2.5.0",
     "coffee-script": "~1.7.0",
     "express": "3.3.4",
-    "logger-sharelatex": "git+https://github.com/sharelatex/logger-sharelatex.git#v1.5.6",
+    "logger-sharelatex": "git+https://github.com/sharelatex/logger-sharelatex.git#v1.5.9",
     "lynx": "0.0.11",
-    "metrics-sharelatex": "git+https://github.com/sharelatex/metrics-sharelatex.git#v2.0.8",
+    "metrics-sharelatex": "git+https://github.com/sharelatex/metrics-sharelatex.git#v2.0.12",
     "redis-sharelatex": "git+https://github.com/sharelatex/redis-sharelatex.git#v1.0.4",
     "request": "2.25.0",
     "requestretry": "^1.12.0",

services/document-updater/test/unit/coffee/ProjectManager/flushAndDeleteProjectTests.coffee

@@ -12,7 +12,7 @@ describe "ProjectManager - flushAndDeleteProject", ->
 			"./DocumentManager": @DocumentManager = {}
 			"logger-sharelatex": @logger = { log: sinon.stub(), error: sinon.stub() }
 			"./HistoryManager": @HistoryManager =
-				flushProjectChangesAsync: sinon.stub()
+				flushProjectChanges: sinon.stub().callsArg(1)
 			"./Metrics": @Metrics =
 				Timer: class Timer
 					done: sinon.stub()
@@ -40,8 +40,8 @@ describe "ProjectManager - flushAndDeleteProject", ->
 					.should.equal true
 
 		it "should flush project history", ->
-			@HistoryManager.flushProjectChangesAsync
-				.calledWithExactly(@project_id)
+			@HistoryManager.flushProjectChanges
+				.calledWith(@project_id)
 				.should.equal true
 
 		it "should call the callback without error", ->
@@ -70,8 +70,8 @@ describe "ProjectManager - flushAndDeleteProject", ->
 					.should.equal true
 
 		it "should still flush project history", ->
-			@HistoryManager.flushProjectChangesAsync
-				.calledWithExactly(@project_id)
+			@HistoryManager.flushProjectChanges
+				.calledWith(@project_id)
 				.should.equal true
 
 		it "should record the error", ->