Tell track changes api to flush doc when flushing doc to mongo

services/document-updater/Gruntfile.coffee

@@ -45,16 +45,16 @@ module.exports = (grunt) ->
 
 		clean:
 			app: ["app/js"]
-			acceptance_tests: ["test/unit/js"]
+			acceptance_tests: ["test/acceptance/js"]
 
 		mochaTest:
 			unit:
-				src: ['test/unit/js/**/*.js']
+				src: ["test/unit/js/#{grunt.option('feature') or '**'}/*.js"]
 				options:
 					reporter: grunt.option('reporter') or 'spec'
 					grep: grunt.option("grep")
 			acceptance:
-				src: ['test/acceptance/js/**/*.js']
+				src: ["test/acceptance/js/#{grunt.option('feature') or '*'}.js"]
 				options:
 					reporter: grunt.option('reporter') or 'spec'
 					grep: grunt.option("grep")

services/document-updater/app.coffee

@@ -21,15 +21,6 @@ app.configure ->
 	app.use express.bodyParser()
 	app.use app.router
 
-app.configure 'development', ()->
-	console.log "Development Enviroment"
-	app.use express.errorHandler({ dumpExceptions: true, showStack: true })
-
-app.configure 'production', ()->
-	console.log "Production Enviroment"
-	app.use express.logger()
-	app.use express.errorHandler()
-
 rclient.subscribe("pending-updates")
 rclient.on "message", (channel, doc_key)->
 	[project_id, doc_id] = Keys.splitProjectIdAndDocId(doc_key)

services/document-updater/app/coffee/DocumentManager.coffee

@@ -2,6 +2,7 @@ RedisManager = require "./RedisManager"
 PersistenceManager = require "./PersistenceManager"
 DocOpsManager = require "./DocOpsManager"
 DiffCodec = require "./DiffCodec"
+TrackChangesManager = require "./TrackChangesManager"
 logger = require "logger-sharelatex"
 Metrics = require "./Metrics"
 
@@ -81,6 +82,10 @@ module.exports = DocumentManager =
 			timer.done()
 			_callback(args...)
 
+		TrackChangesManager.flushDocChanges doc_id, (error) ->
+			if error?
+				logger.error err: error, project_id: project_id, doc_id: doc_id, "error flushing doc to track changes api"
+
 		RedisManager.getDoc doc_id, (error, lines, version) ->
 			return callback(error) if error?
 			if !lines? or !version?

services/document-updater/app/coffee/TrackChangesManager.coffee

@@ -0,0 +1,20 @@
+settings = require "settings-sharelatex"
+request  = require "request"
+logger = require "logger-sharelatex"
+
+module.exports =
+	flushDocChanges: (doc_id, callback = (error) ->) ->
+		if !settings.apis?.trackchanges?
+			logger.warn doc_id: doc_id, "track changes API is not configured, so not flushing"
+			return callback()
+
+		url = "#{settings.apis.trackchanges.url}/doc/#{doc_id}/flush"
+		logger.log doc_id: doc_id, url: url, "flushing doc in track changes api"
+		request.post url, (error, res, body)->
+			if error?
+				return callback(error)
+			else if res.statusCode >= 200 and res.statusCode < 300
+				return callback(null)
+			else
+				error = new Error("track changes api returned a failure status code: #{res.statusCode}")
+				return callback(error)

services/document-updater/config/settings.development.coffee

@@ -12,6 +12,8 @@ module.exports =
 			url: "http://localhost:3000"
 			user: "sharelatex"
 			pass: "password"
+		trackchanges:
+			url: "http://localhost:3014"
 
 	redis:
 		web:

services/document-updater/test/acceptance/coffee/ApplyingUpdatesToADocTests.coffee

@@ -5,6 +5,7 @@ async = require "async"
 mongojs = require "../../../app/js/mongojs"
 db = mongojs.db
 ObjectId = mongojs.ObjectId
+rclient = require("redis").createClient()
 
 MockWebApi = require "./helpers/MockWebApi"
 DocUpdaterClient = require "./helpers/DocUpdaterClient"
@@ -45,6 +46,11 @@ describe "Applying updates to a doc", ->
 				doc.lines.should.deep.equal @result
 				done()
 
+		it "should push the applied updates to the track changes api", (done) ->
+			rclient.lrange "UncompressedHistoryOps:#{@doc_id}", 0, -1, (error, updates) =>
+				JSON.parse(updates[0]).op.should.deep.equal @update.op
+				done()
+
 	describe "when the document is loaded", ->
 		before (done) ->
 			[@project_id, @doc_id] = [DocUpdaterClient.randomId(), DocUpdaterClient.randomId()]
@@ -69,6 +75,11 @@ describe "Applying updates to a doc", ->
 				doc.lines.should.deep.equal @result
 				done()
 
+		it "should push the applied updates to the track changes api", (done) ->
+			rclient.lrange "UncompressedHistoryOps:#{@doc_id}", 0, -1, (error, updates) =>
+				JSON.parse(updates[0]).op.should.deep.equal @update.op
+				done()
+
 	describe "when the document has been deleted", ->
 		describe "when the ops come in a single linear order", ->
 			before ->
@@ -112,6 +123,13 @@ describe "Applying updates to a doc", ->
 						doc.lines.should.deep.equal @result
 						done()
 
+			it "should push the applied updates to the track changes api", (done) ->
+				rclient.lrange "UncompressedHistoryOps:#{@doc_id}", 0, -1, (error, updates) =>
+					updates = (JSON.parse(u) for u in updates)
+					for appliedUpdate, i in @updates
+						appliedUpdate.op.should.deep.equal updates[i].op
+					done()
+
 		describe "when older ops come in after the delete", ->
 			before ->
 				[@project_id, @doc_id] = [DocUpdaterClient.randomId(), DocUpdaterClient.randomId()]

services/document-updater/test/acceptance/coffee/FlushingAProjectTests.coffee

@@ -4,6 +4,7 @@ chai.should()
 async = require "async"
 
 MockWebApi = require "./helpers/MockWebApi"
+MockTrackChangesApi = require "./helpers/MockTrackChangesApi"
 DocUpdaterClient = require "./helpers/DocUpdaterClient"
 
 describe "Flushing a project", ->
@@ -40,6 +41,8 @@ describe "Flushing a project", ->
 	describe "with documents which have been updated", ->
 		before (done) ->
 			sinon.spy MockWebApi, "setDocumentLines"
+			sinon.spy MockTrackChangesApi, "flushDoc"
+
 			async.series @docs.map((doc) =>
 				(callback) =>
 					DocUpdaterClient.preloadDoc @project_id, doc.id, (error) =>
@@ -56,6 +59,7 @@ describe "Flushing a project", ->
 
 		after ->
 			MockWebApi.setDocumentLines.restore()
+			MockTrackChangesApi.flushDoc.restore()
 
 		it "should return a 204 status code", ->
 			@statusCode.should.equal 204
@@ -74,3 +78,13 @@ describe "Flushing a project", ->
 						callback()
 			), done
 
+		it "should flush the docs in the track changes api", (done) ->
+			# This is done in the background, so wait a little while to ensure it has happened
+			setTimeout () =>
+				async.series @docs.map((doc) =>
+					(callback) =>
+						MockTrackChangesApi.flushDoc.calledWith(doc.id).should.equal true
+				), done
+				done()
+			, 100
+

services/document-updater/test/acceptance/coffee/FlushingDocsTests.coffee

@@ -4,6 +4,7 @@ chai.should()
 async = require "async"
 
 MockWebApi = require "./helpers/MockWebApi"
+MockTrackChangesApi = require "./helpers/MockTrackChangesApi"
 DocUpdaterClient = require "./helpers/DocUpdaterClient"
 mongojs = require "../../../app/js/mongojs"
 db = mongojs.db
@@ -31,6 +32,7 @@ describe "Flushing a doc to Mongo", ->
 				lines: @lines
 			}
 			sinon.spy MockWebApi, "setDocumentLines"
+			sinon.spy MockTrackChangesApi, "flushDoc"
 
 			DocUpdaterClient.sendUpdates @project_id, @doc_id, [@update], (error) =>
 				throw error if error?
@@ -40,6 +42,7 @@ describe "Flushing a doc to Mongo", ->
 
 		after ->
 			MockWebApi.setDocumentLines.restore()
+			MockTrackChangesApi.flushDoc.restore()
 
 		it "should flush the updated document to the web api", ->
 			MockWebApi.setDocumentLines
@@ -52,6 +55,13 @@ describe "Flushing a doc to Mongo", ->
 				doc.docOps[0].op.should.deep.equal @update.op
 				done()
 
+		it "should flush the doc in the track changes api", (done) ->
+			# This is done in the background, so wait a little while to ensure it has happened
+			setTimeout () =>
+				MockTrackChangesApi.flushDoc.calledWith(@doc_id).should.equal true
+				done()
+			, 100
+
 	describe "when the doc has a large number of ops to be flushed", ->
 		before (done) ->
 			[@project_id, @doc_id] = [DocUpdaterClient.randomId(), DocUpdaterClient.randomId()]
@@ -93,5 +103,5 @@ describe "Flushing a doc to Mongo", ->
 
 		it "should not flush the doc to the web api", ->
 			MockWebApi.setDocumentLines.called.should.equal false
-			
+
 

services/document-updater/test/acceptance/coffee/helpers/MockTrackChangesApi.coffee

@@ -0,0 +1,20 @@
+express = require("express")
+app = express()
+
+module.exports = MockTrackChangesApi =
+	flushDoc: (doc_id, callback = (error) ->) ->
+		callback()
+
+	run: () ->
+		app.post "/doc/:doc_id/flush", (req, res, next) =>
+			@flushDoc req.params.doc_id, (error) ->
+				if error?
+					res.send 500
+				else
+					res.send 204
+
+		app.listen 3014, (error) ->
+			throw error if error?
+
+MockTrackChangesApi.run()
+

services/document-updater/test/acceptance/coffee/helpers/MockWebApi.coffee

@@ -34,7 +34,8 @@ module.exports = MockWebApi =
 				else
 					res.send 204
 
-		app.listen(3000)
+		app.listen 3000, (error) ->
+			throw error if error?
 
 MockWebApi.run()
 

services/document-updater/test/unit/coffee/DocumentManager/flushDocTests.coffee

@@ -10,6 +10,7 @@ describe "DocumentUpdater - flushDocIfLoaded", ->
 			"./RedisManager": @RedisManager = {}
 			"./PersistenceManager": @PersistenceManager = {}
 			"./DocOpsManager": @DocOpsManager = {}
+			"./TrackChangesManager": @TrackChangesManager = {}
 			"logger-sharelatex": @logger = {log: sinon.stub()}
 			"./Metrics": @Metrics =
 				Timer: class Timer
@@ -25,6 +26,7 @@ describe "DocumentUpdater - flushDocIfLoaded", ->
 			@RedisManager.getDoc = sinon.stub().callsArgWith(1, null, @lines, @version)
 			@PersistenceManager.setDoc = sinon.stub().callsArgWith(3)
 			@DocOpsManager.flushDocOpsToMongo = sinon.stub().callsArgWith(2)
+			@TrackChangesManager.flushDocChanges = sinon.stub().callsArg(1)
 			@DocumentManager.flushDocIfLoaded @project_id, @doc_id, @callback
 
 		it "should get the doc from redis", ->
@@ -48,11 +50,17 @@ describe "DocumentUpdater - flushDocIfLoaded", ->
 		it "should time the execution", ->
 			@Metrics.Timer::done.called.should.equal true
 
+		it "should flush the doc in the track changes api", ->
+			@TrackChangesManager.flushDocChanges
+				.calledWith(@doc_id)
+				.should.equal true
+
 	describe "when the document is not in Redis", ->
 		beforeEach ->
 			@RedisManager.getDoc = sinon.stub().callsArgWith(1, null, null, null)
 			@PersistenceManager.setDoc = sinon.stub().callsArgWith(3)
 			@DocOpsManager.flushDocOpsToMongo = sinon.stub().callsArgWith(2)
+			@TrackChangesManager.flushDocChanges = sinon.stub().callsArg(1)
 			@DocumentManager.flushDocIfLoaded @project_id, @doc_id, @callback
 
 		it "should get the doc from redis", ->

services/document-updater/test/unit/coffee/TrackChangesManager/TrackChangesManagerTests.coffee

@@ -0,0 +1,38 @@
+SandboxedModule = require('sandboxed-module')
+sinon = require('sinon')
+require('chai').should()
+modulePath = require('path').join __dirname, '../../../../app/js/TrackChangesManager'
+
+describe "TrackChangesManager", ->
+	beforeEach ->
+		@TrackChangesManager = SandboxedModule.require modulePath, requires:
+			"request": @request = {}
+			"settings-sharelatex": @Settings = {}
+		@doc_id = "mock-doc-id"
+		@callback = sinon.stub()
+
+	describe "flushDocChanges", ->
+		beforeEach ->
+			@Settings.apis =
+				trackchanges: url: "http://trackchanges.example.com"
+
+		describe "successfully", ->
+			beforeEach ->
+				@request.post = sinon.stub().callsArgWith(1, null, statusCode: 204)
+				@TrackChangesManager.flushDocChanges @doc_id, @callback
+
+			it "should send a request to the track changes api", ->
+				@request.post
+					.calledWith("#{@Settings.apis.trackchanges.url}/doc/#{@doc_id}/flush")
+					.should.equal true
+
+			it "should return the callback", ->
+				@callback.calledWith(null).should.equal true
+
+		describe "when the track changes api returns an error", ->
+			beforeEach ->
+				@request.post = sinon.stub().callsArgWith(1, null, statusCode: 500)
+				@TrackChangesManager.flushDocChanges @doc_id, @callback
+
+			it "should return the callback with an error", ->
+				@callback.calledWith(new Error("track changes api return non-success code: 500")).should.equal true