Use Dropbox Real-time polling

services/web/app/coffee/Features/Dropbox/DropboxRouter.coffee

@@ -0,0 +1,15 @@
+DropboxUserController = require './DropboxUserController'
+DropboxWebhookController = require './DropboxWebhookController'
+
+module.exports =
+	apply: (app) ->
+		app.get  '/dropbox/beginAuth', DropboxUserController.redirectUserToDropboxAuth
+		app.get  '/dropbox/completeRegistration', DropboxUserController.completeDropboxRegistration
+		app.get  '/dropbox/unlink', DropboxUserController.unlinkDropbox
+		
+		app.get  '/dropbox/webhook', DropboxWebhookController.verify
+		app.post '/dropbox/webhook', DropboxWebhookController.webhook
+		app.ignoreCsrf('post', '/dropbox/webhook')
+
+
+

services/web/app/coffee/Features/Dropbox/DropboxWebhookController.coffee

@@ -0,0 +1,15 @@
+logger = require("logger-sharelatex")
+DropboxWebhookHandler = require("./DropboxWebhookHandler")
+
+module.exports = DropboxWebhookController =
+	verify: (req, res, next = (error) ->) ->
+		res.send(req.query.challenge)
+		
+	webhook: (req, res, next = (error) ->) ->
+		dropbox_uids = req.body?.delta?.users
+		logger.log dropbox_uids: dropbox_uids, "received webhook request from Dropbox"
+		if !dropbox_uids?
+			return res.send(400) # Bad Request
+		DropboxWebhookHandler.pollDropboxUids dropbox_uids, (error) ->
+			return next(error) if error?
+			res.send(200)

services/web/app/coffee/Features/Dropbox/DropboxWebhookHandler.coffee

@@ -0,0 +1,25 @@
+logger = require("logger-sharelatex")
+async = require "async"
+User = require("../../models/User").User
+TpdsUpdateSender = require "../ThirdPartyDataStore/TpdsUpdateSender"
+
+module.exports = DropboxWebhookHandler =
+	pollDropboxUids: (dropbox_uids, callback = (error) ->) ->
+		jobs = []
+		for uid in dropbox_uids
+			do (uid) ->
+				jobs.push (callback) ->
+					DropboxWebhookHandler.pollDropboxUid uid, callback
+		async.series jobs, callback
+		
+	pollDropboxUid: (dropbox_uid, callback = (error) ->) ->
+		User.find {
+			"dropbox.access_token.uid": dropbox_uid
+			"features.dropbox": true
+		}, (error, users = []) ->
+			return callback(error) if error?
+			user = users[0]
+			if !user?
+				logger.log dropbox_uid: dropbox_uid, "no sharelatex user found"
+				return callback()
+			TpdsUpdateSender.pollDropboxForUser user._id, callback

services/web/app/coffee/Features/ServerAdmin/AdminController.coffee

@@ -73,10 +73,6 @@ module.exports = AdminController =
 	flushProjectToTpds: (req, res)->
 		projectEntityHandler.flushProjectToThirdPartyDataStore req.body.project_id, (err)->
 			res.send 200
-
-	pollUsersWithDropbox: (req, res)->
-		TpdsPollingBackgroundTasks.pollUsersWithDropbox ->
-			res.send 200
 			
 	createMessage: (req, res, next) ->
 		SystemMessageManager.createMessage req.body.content, (error) ->

services/web/app/coffee/Features/ThirdPartyDataStore/TpdsUpdateSender.coffee

@@ -87,7 +87,16 @@ module.exports =
 				title:"deleteEntity"
 				sl_all_user_ids:JSON.stringify(allUserIds)
 			queue.enqueue options.project_id, "standardHttpRequest", deleteOptions, callback
-
+			
+	pollDropboxForUser: (user_id, callback = (err) ->) ->
+		metrics.inc("tpds.poll-dropbox")
+		logger.log user_id: user_id, "polling dropbox for user"
+		options =
+			method: "POST"
+			uri:"#{settings.apis.thirdPartyDataStore.url}/user/poll"
+			json:
+				user_ids: [user_id]
+		queue.enqueue "poll-dropbox:#{user_id}", "standardHttpRequest", options, callback
 
 getProjectsUsersIds = (project_id, callback = (err, owner_id, allUserIds)->)->
 	Project.findById project_id, "_id owner_ref readOnly_refs collaberator_refs", (err, project)->

services/web/app/coffee/router.coffee

@@ -10,7 +10,6 @@ EditorHttpController = require("./Features/Editor/EditorHttpController")
 EditorUpdatesController = require("./Features/Editor/EditorUpdatesController")
 Settings = require('settings-sharelatex')
 TpdsController = require('./Features/ThirdPartyDataStore/TpdsController')
-dropboxHandler = require('./Features/Dropbox/DropboxHandler')
 SubscriptionRouter = require './Features/Subscription/SubscriptionRouter'
 UploadsRouter = require './Features/Uploads/UploadsRouter'
 metrics = require('./infrastructure/Metrics')
@@ -32,13 +31,14 @@ HealthCheckController = require("./Features/HealthCheck/HealthCheckController")
 ProjectDownloadsController = require "./Features/Downloads/ProjectDownloadsController"
 FileStoreController = require("./Features/FileStore/FileStoreController")
 TrackChangesController = require("./Features/TrackChanges/TrackChangesController")
-DropboxUserController = require("./Features/Dropbox/DropboxUserController")
 PasswordResetRouter = require("./Features/PasswordReset/PasswordResetRouter")
 StaticPagesRouter = require("./Features/StaticPages/StaticPagesRouter")
 ChatController = require("./Features/Chat/ChatController")
 BlogController = require("./Features/Blog/BlogController")
 WikiController = require("./Features/Wiki/WikiController")
 ConnectedUsersController = require("./Features/ConnectedUsers/ConnectedUsersController")
+DropboxRouter = require "./Features/Dropbox/DropboxRouter"
+dropboxHandler = require "./Features/Dropbox/DropboxHandler"
 
 logger = require("logger-sharelatex")
 _ = require("underscore")
@@ -66,6 +66,7 @@ module.exports = class Router
 		PasswordResetRouter.apply(app)
 		StaticPagesRouter.apply(app)
 		TemplatesRouter.apply(app)
+		DropboxRouter.apply(app)
 
 		app.get '/blog', BlogController.getIndexPage
 		app.get '/blog/*', BlogController.getPage
@@ -80,10 +81,6 @@ module.exports = class Router
 		app.del  '/user/newsletter/unsubscribe', AuthenticationController.requireLogin(), UserController.unsubscribe
 		app.del  '/user', AuthenticationController.requireLogin(), UserController.deleteUser
 
-		app.get  '/dropbox/beginAuth', DropboxUserController.redirectUserToDropboxAuth
-		app.get  '/dropbox/completeRegistration', DropboxUserController.completeDropboxRegistration
-		app.get  '/dropbox/unlink', DropboxUserController.unlinkDropbox
-
 		app.get  '/user/auth_token', AuthenticationController.requireLogin(), AuthenticationController.getAuthToken
 		app.get  '/user/personal_info', AuthenticationController.requireLogin(allow_auth_token: true), UserInfoController.getLoggedInUsersPersonalInfo
 		app.get  '/user/:user_id/personal_info', httpAuth, UserInfoController.getPersonalInfo
@@ -172,7 +169,6 @@ module.exports = class Router
 		app.post '/admin/dissconectAllUsers', SecurityManager.requestIsAdmin, AdminController.dissconectAllUsers
 		app.post '/admin/syncUserToSubscription', SecurityManager.requestIsAdmin, AdminController.syncUserToSubscription
 		app.post '/admin/flushProjectToTpds', SecurityManager.requestIsAdmin, AdminController.flushProjectToTpds
-		app.post '/admin/pollUsersWithDropbox', SecurityManager.requestIsAdmin, AdminController.pollUsersWithDropbox
 		app.post '/admin/messages', SecurityManager.requestIsAdmin, AdminController.createMessage
 		app.post '/admin/messages/clear', SecurityManager.requestIsAdmin, AdminController.clearMessages
 

services/web/app/views/admin.jade

@@ -55,11 +55,6 @@ block content
 											input.form-control(type='text',  name='project_id', placeholder='project_id', required)
 										.form-group
 											button.btn-primary.btn(type='submit') Flush
-								hr
-								.row-spaced
-									form(enctype='multipart/form-data', method='post',action='/admin/pollUsersWithDropbox')
-										input(name="_csrf", type="hidden", value=csrfToken)
-										button.btn.btn-primary(type="submit") Poll users with dropbox
 										
 							tab(heading="System Messages")
 								each message in systemMessages 

services/web/app/views/project/editor/dropbox.jade

@@ -20,10 +20,6 @@ script(type="text/ng-template", id="dropboxModalTemplate")
 
 
 			div(ng-show="dbState.hasDropboxFeature && dbState.userIsLinkedToDropbox")
-				progressbar.progress-striped.active(value='dbState.percentageLeftTillNextPoll', type="info")
-				p
-					strong {{dbState.minsTillNextPoll}} #{translate("minutes")} 
-					span #{translate("until_db_checked_for_changes")}
 				p.small
 					| #{translate("this_project_will_appear_in_your_dropbox_folder_at")}
 					strong Dropbox/sharelatex/{{ project.name }}

services/web/public/coffee/ide/dropbox/controllers/DropboxController.coffee

@@ -27,20 +27,10 @@ define [
 
 		$scope.dbState = cachedState
 		$scope.dbState.hasDropboxFeature = $scope.project.features.dropbox
-		
-		calculatePollTime = ->
-			ide.socket.emit "getLastTimePollHappned", (err, lastTimePollHappened)=>
-				milisecondsSinceLastPoll = new Date().getTime() - lastTimePollHappened
-				roundedMinsSinceLastPoll = Math.round(milisecondsSinceLastPoll / ONE_MIN_MILI)
-
-				$scope.dbState.minsTillNextPoll = POLLING_INTERVAL - roundedMinsSinceLastPoll
-				$scope.dbState.percentageLeftTillNextPoll = ((roundedMinsSinceLastPoll / POLLING_INTERVAL) * 100)
-				$timeout calculatePollTime, 60 * 1000
 
 		ide.socket.emit "getUserDropboxLinkStatus", user_id, (err, status)=>
 			$scope.dbState.gotLinkStatus = true
-			if status.registered 
+			if status.registered
-				calculatePollTime()
 				$scope.dbState.userIsLinkedToDropbox = true
 				cachedState = $scope.dbState
 		

services/web/test/UnitTests/coffee/Dropbox/DropboxWebhookControllerTests.coffee

@@ -0,0 +1,48 @@
+SandboxedModule = require('sandboxed-module')
+assert = require('assert')
+require('chai').should()
+sinon = require('sinon')
+modulePath = require('path').join __dirname, '../../../../app/js/Features/Dropbox/DropboxWebhookController.js'
+
+describe 'DropboxWebhookController', ->
+	beforeEach ->
+
+		@DropboxWebhookController = SandboxedModule.require modulePath, requires:
+			"./DropboxWebhookHandler": @DropboxWebhookHandler = {}
+			'logger-sharelatex':
+				log:->
+				err:->
+
+	describe "verify", ->
+		beforeEach ->
+			@res =
+				send: sinon.stub()
+			@req.query =
+				challenge: @challenge = "foo"
+			@DropboxWebhookController.verify(@req, @res)
+			
+		it "should echo the challenge parameter back", ->
+			@res.send.calledWith(@challenge).should.equal true
+			
+	describe "webhook", ->
+		beforeEach ->
+			@req.body =
+				delta:
+					users: @dropbox_uids = [
+						"123456",
+						"789123"
+					]
+			@res.send = sinon.stub()
+			@DropboxWebhookHandler.pollDropboxUids = sinon.stub().callsArg(1)
+			@DropboxWebhookController.webhook(@req, @res)
+			
+		it "should poll the Dropbox uids", ->
+			@DropboxWebhookHandler.pollDropboxUids
+				.calledWith(@dropbox_uids)
+				.should.equal true
+				
+		it "should return success", ->
+			@res.send
+				.calledWith(200)
+				.should.equal true
+

services/web/test/UnitTests/coffee/Dropbox/DropboxWebhookHandlerTests.coffee

@@ -0,0 +1,51 @@
+SandboxedModule = require('sandboxed-module')
+assert = require('assert')
+require('chai').should()
+sinon = require('sinon')
+modulePath = require('path').join __dirname, '../../../../app/js/Features/Dropbox/DropboxWebhookHandler.js'
+
+describe 'DropboxWebhookHandler', ->
+	beforeEach ->
+		@DropboxWebhookHandler = SandboxedModule.require modulePath, requires:
+			"../../models/User": User: @User = {}
+			"../ThirdPartyDataStore/TpdsUpdateSender": @TpdsUpdateSender = {}
+			'logger-sharelatex':
+				log:->
+				err:->
+		@callback = sinon.stub()
+
+	describe "pollDropboxUids", ->
+		beforeEach (done) ->
+			@dropbox_uids = [
+				"123456",
+				"789123"
+			]
+			@DropboxWebhookHandler.pollDropboxUid = sinon.stub().callsArg(1)
+			@DropboxWebhookHandler.pollDropboxUids @dropbox_uids, done
+			
+		it "should call pollDropboxUid for each uid", ->
+			for uid in @dropbox_uids
+				@DropboxWebhookHandler.pollDropboxUid
+					.calledWith(uid)
+					.should.equal true
+					
+	describe "pollDropboxUid", ->
+		beforeEach ->
+			@dropbox_uid = "dropbox-123456"
+			@user_id = "sharelatex-user-id"
+			@User.find = sinon.stub().callsArgWith(1, null, [ _id: @user_id ])
+			@TpdsUpdateSender.pollDropboxForUser = sinon.stub().callsArg(1)
+			@DropboxWebhookHandler.pollDropboxUid @dropbox_uid, @callback
+			
+		it "should look up the user", ->
+			@User.find
+				.calledWith({ "dropbox.access_token.uid": @dropbox_uid, "features.dropbox": true })
+				.should.equal true
+				
+		it "should poll the user's Dropbox", ->
+			@TpdsUpdateSender.pollDropboxForUser
+				.calledWith(@user_id)
+				.should.equal true
+				
+		it "should call the callback", ->
+			@callback.called.should.equal true

services/web/test/UnitTests/coffee/ThirdPartyDataStore/TpdsUpdateSenderTests.coffee

@@ -105,3 +105,20 @@ describe 'TpdsUpdateSender', ->
 				job.headers.sl_all_user_ids.should.eql(JSON.stringify([collaberator_ref_1, read_only_ref_1, user_id]))
 				done()
 			@updateSender.moveEntity {project_id:project_id, project_name:oldProjectName, newProjectName:newProjectName}
+			
+		it "pollDropboxForUser", (done) ->
+			@requestQueuer.enqueue = sinon.stub().callsArg(3)
+			@updateSender.pollDropboxForUser user_id, (error) =>
+				@requestQueuer.enqueue
+					.calledWith(
+						"poll-dropbox:#{user_id}",
+						"standardHttpRequest",
+						{
+							method: "POST"
+							uri: "#{thirdPartyDataStoreApiUrl}/user/poll"
+							json:
+								user_ids: [user_id]
+						}
+					)
+					.should.equal true
+				done()