From 0a44aa9e71ab6fc11ee514b488532d09b0d1732f Mon Sep 17 00:00:00 2001 From: James Allen Date: Thu, 7 Aug 2014 13:19:10 +0100 Subject: [PATCH 01/12] Push into doc updater queue rather than pub/sub for updates --- .../DocumentUpdaterHandler.coffee | 12 ++++--- .../DocumentUpdaterHandlerTests.coffee | 32 ++++++------------- 2 files changed, 16 insertions(+), 28 deletions(-) diff --git a/services/web/app/coffee/Features/DocumentUpdater/DocumentUpdaterHandler.coffee b/services/web/app/coffee/Features/DocumentUpdater/DocumentUpdaterHandler.coffee index 3d4d49c827..c5fcc11209 100644 --- a/services/web/app/coffee/Features/DocumentUpdater/DocumentUpdaterHandler.coffee +++ b/services/web/app/coffee/Features/DocumentUpdater/DocumentUpdaterHandler.coffee @@ -18,12 +18,14 @@ module.exports = DocumentUpdaterHandler = queueChange : (project_id, doc_id, change, sl_req_id, callback = ()->)-> {callback, sl_req_id} = slReqIdHelper.getCallbackAndReqId(callback, sl_req_id) jsonChange = JSON.stringify change - rclient.rpush keys.pendingUpdates(doc_id:doc_id), jsonChange, (error)-> + doc_key = keys.combineProjectIdAndDocId(project_id, doc_id) + multi = rclient.multi() + multi.rpush keys.pendingUpdates(doc_id:doc_id), jsonChange + multi.sadd keys.docsWithPendingUpdates, doc_key + multi.rpush "pending-updates-list", doc_key + multi.exec (error) -> return callback(error) if error? - doc_key = keys.combineProjectIdAndDocId(project_id, doc_id) - rclient.sadd keys.docsWithPendingUpdates, doc_key, (error) -> - return callback(error) if error? - rclient.publish "pending-updates", doc_key, callback + callback() flushProjectToMongo: (project_id, sl_req_id, callback = (error) ->)-> {callback, sl_req_id} = slReqIdHelper.getCallbackAndReqId(callback, sl_req_id) diff --git a/services/web/test/UnitTests/coffee/DocumentUpdater/DocumentUpdaterHandlerTests.coffee b/services/web/test/UnitTests/coffee/DocumentUpdater/DocumentUpdaterHandlerTests.coffee index 097269a731..f0bd52f03e 100644 --- a/services/web/test/UnitTests/coffee/DocumentUpdater/DocumentUpdaterHandlerTests.coffee +++ b/services/web/test/UnitTests/coffee/DocumentUpdater/DocumentUpdaterHandlerTests.coffee @@ -40,9 +40,10 @@ describe 'Flushing documents :', -> "range":{"start":{"row":2,"column":2},"end":{"row":2,"column":3}}, "text":"e" } - @rclient.rpush = sinon.stub().callsArg(2) - @rclient.publish = sinon.stub().callsArg(2) - @rclient.sadd = sinon.stub().callsArg(2) + @rclient.multi = sinon.stub().returns @rclient + @rclient.exec = sinon.stub().callsArg(0) + @rclient.rpush = sinon.stub() + @rclient.sadd = sinon.stub() @callback = sinon.stub() describe "successfully", -> @@ -54,9 +55,9 @@ describe 'Flushing documents :', -> .calledWith("PendingUpdates:#{@doc_id}", JSON.stringify(@change)) .should.equal true - it "should notify the doc updater of the change via pub/sub", -> - @rclient.publish - .calledWith("pending-updates", "#{@project_id}:#{@doc_id}") + it "should notify the doc updater of the change via the pending-updates-list queue", -> + @rclient.rpush + .calledWith("pending-updates-list", "#{@project_id}:#{@doc_id}") .should.equal true it "should push the doc id into the pending updates set", -> @@ -64,29 +65,14 @@ describe 'Flushing documents :', -> .calledWith("DocsWithPendingUpdates", "#{@project_id}:#{@doc_id}") .should.equal true - describe "with error connecting to redis during push", -> + describe "with error connecting to redis during exec", -> beforeEach -> - @rclient.rpush = sinon.stub().callsArgWith(2, new Error("something went wrong")) + @rclient.exec = sinon.stub().callsArgWith(0, new Error("something went wrong")) @handler.queueChange(@project_id, @doc_id, @change, @callback) it "should return an error", -> @callback.calledWithExactly(sinon.match(Error)).should.equal true - describe "with error connecting to redis during publish", -> - beforeEach -> - @rclient.publish = sinon.stub().callsArgWith(2, new Error("something went wrong")) - @handler.queueChange(@project_id, @doc_id, @change, @callback) - - it "should return an error", -> - @callback.calledWithExactly(sinon.match(Error)).should.equal true - - describe "with error connecting to redis during sadd", -> - beforeEach -> - @rclient.sadd = sinon.stub().callsArgWith(2, new Error("something went wrong")) - @handler.queueChange(@project_id, @doc_id, @change, @callback) - - it "should return an error", -> - @callback.calledWithExactly(sinon.match(Error)).should.equal true describe 'flushProjectToMongo', -> beforeEach -> From 2aa421233cfd01d98d9dfa30fc864d5098a34e03 Mon Sep 17 00:00:00 2001 From: James Allen Date: Mon, 11 Aug 2014 16:20:27 +0100 Subject: [PATCH 02/12] Add meta description for templates home page --- services/web/app/views/templates/index.jade | 3 +++ 1 file changed, 3 insertions(+) diff --git a/services/web/app/views/templates/index.jade b/services/web/app/views/templates/index.jade index d265af4f36..5fe4d070f5 100644 --- a/services/web/app/views/templates/index.jade +++ b/services/web/app/views/templates/index.jade @@ -1,5 +1,8 @@ extends ../layout +block vars + - var meta = "Over 400 LaTeX templates for journal articles, theses, CV and resumes, posters, presentations, and much more" + block content .content.content-alt .container From 3348249dcdb89a7afdcc866b86cf32044c1c0284 Mon Sep 17 00:00:00 2001 From: James Allen Date: Tue, 12 Aug 2014 12:02:09 +0100 Subject: [PATCH 03/12] Download template PDF and zip file with good file names --- .../Features/Templates/TemplatesWebController.coffee | 12 +++++++++++- services/web/app/views/templates/template.jade | 6 +++--- services/web/public/stylesheets/app/templates.less | 6 ++++++ 3 files changed, 20 insertions(+), 4 deletions(-) diff --git a/services/web/app/coffee/Features/Templates/TemplatesWebController.coffee b/services/web/app/coffee/Features/Templates/TemplatesWebController.coffee index 19cb7c5fe3..aa6952a8a3 100644 --- a/services/web/app/coffee/Features/Templates/TemplatesWebController.coffee +++ b/services/web/app/coffee/Features/Templates/TemplatesWebController.coffee @@ -36,7 +36,17 @@ module.exports = TemplatesWebController = proxyToTemplatesApi: (req, res)-> url = req.url - logger.log url:url, "proxying request to templates api" + + name = req.query.name or "Template" + if req.query.inline? + disposition = "inline" + else + disposition = "attachment" + console.log "HEADER", "#{disposition}; filename=#{name};" + res.header({"content-disposition": "#{disposition}; filename=#{name}.#{req.params.file_type};"}) + + logger.log url:url, template_name: name, disposition: disposition, "proxying request to templates api" + getReq = request.get("#{settings.apis.templates_api.url}#{url}") getReq.pipe(res) getReq.on "error", (error) -> diff --git a/services/web/app/views/templates/template.jade b/services/web/app/views/templates/template.jade index c5047071ad..f834d9c5ee 100644 --- a/services/web/app/views/templates/template.jade +++ b/services/web/app/views/templates/template.jade @@ -20,8 +20,8 @@ block content .col-md-6 .entry .row - .col-md-12 - a(href=template.pdfUrl) + .col-md-12.template-large-pdf-preview + a(href="#{template.pdfUrl}?inline=true&name=#{template.name}") img(src="#{template.previewUrl}") .col-md-6 @@ -33,7 +33,7 @@ block content |   a.btn.btn-default( - href=template.zipUrl, + href="#{template.zipUrl}?name=#{template.name}", rel='nofollow', ng-click='downloadZip()', tooltip-placement="bottom", diff --git a/services/web/public/stylesheets/app/templates.less b/services/web/public/stylesheets/app/templates.less index ffd3c6cf85..eeb90592d1 100644 --- a/services/web/public/stylesheets/app/templates.less +++ b/services/web/public/stylesheets/app/templates.less @@ -56,4 +56,10 @@ max-width: 100%; height: auto; } +} + +.template-large-pdf-preview { + img { + max-width: 100%; + } } \ No newline at end of file From 5b3bf5c956a2fc176cb36a61a6e5925c908f9d12 Mon Sep 17 00:00:00 2001 From: James Allen Date: Tue, 12 Aug 2014 12:30:07 +0100 Subject: [PATCH 04/12] Spaces after error titles --- services/web/app/views/project/editor/pdf.jade | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/services/web/app/views/project/editor/pdf.jade b/services/web/app/views/project/editor/pdf.jade index 4b1600e15b..4af1da3b74 100644 --- a/services/web/app/views/project/editor/pdf.jade +++ b/services/web/app/views/project/editor/pdf.jade @@ -80,12 +80,12 @@ div.full-size.pdf(ng-controller="PdfController") .pdf-errors(ng-show="pdf.timedout || pdf.error") .alert.alert-danger(ng-show="pdf.error") - strong #{translate("server_error")} + strong #{translate("server_error")} span #{translate("somthing_went_wrong_compiling")} .alert.alert-danger(ng-show="pdf.timedout") p - strong #{translate("timedout")}. + strong #{translate("timedout")}. span #{translate("proj_timed_out_reason")} p a.text-info(href="https://www.sharelatex.com/learn/Debugging_Compilation_timeout_errors", target="_blank") From f902cfdcf6f03338da9fb617cf15d5d2ef5ff47a Mon Sep 17 00:00:00 2001 From: James Allen Date: Tue, 12 Aug 2014 12:33:12 +0100 Subject: [PATCH 05/12] Monkey patch ace until cursor patches come from upstream --- services/web/public/js/ace/ace.js | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/services/web/public/js/ace/ace.js b/services/web/public/js/ace/ace.js index 818cd4b7a8..25f11ec327 100644 --- a/services/web/public/js/ace/ace.js +++ b/services/web/public/js/ace/ace.js @@ -11428,6 +11428,7 @@ var Editor = function(renderer, session) { this._signal("change", e); this.$cursorChange(); + this.$updateHighlightActiveLine(); }; this.onTokenizerUpdate = function(e) { @@ -14853,8 +14854,10 @@ var VirtualRenderer = function(container, theme) { this.$changedLines.lastRow = lastRow; } - if (this.$changedLines.firstRow > this.layerConfig.lastRow || - this.$changedLines.lastRow < this.layerConfig.firstRow) + if (this.$changedLines.lastRow < this.layerConfig.firstRow) + this.$changedLines.lastRow = this.layerConfig.lastRow + + if (this.$changedLines.firstRow > this.layerConfig.lastRow) return; this.$loop.schedule(this.CHANGE_LINES); }; @@ -15218,6 +15221,11 @@ var VirtualRenderer = function(container, theme) { changes & this.CHANGE_H_SCROLL ) { changes |= this.$computeLayerConfig(); + if (config.firstRow != this.layerConfig.firstRow && config.firstRowScreen == this.layerConfig.firstRowScreen) { + this.scrollTop = this.scrollTop + (config.firstRow - this.layerConfig.firstRow) * this.lineHeight; + changes = changes | this.CHANGE_SCROLL; + changes |= this.$computeLayerConfig(); + } config = this.layerConfig; this.$updateScrollBarV(); if (changes & this.CHANGE_H_SCROLL) From 93f2969be5f743039421c423b9f37353e1da1715 Mon Sep 17 00:00:00 2001 From: James Allen Date: Tue, 12 Aug 2014 12:46:05 +0100 Subject: [PATCH 06/12] Change style on file rename input to avoid weird Chrome behaviour --- .../app/views/project/editor/file-tree.jade | 34 ++++++++++--------- .../stylesheets/app/editor/file-tree.less | 12 +++++++ 2 files changed, 30 insertions(+), 16 deletions(-) diff --git a/services/web/app/views/project/editor/file-tree.jade b/services/web/app/views/project/editor/file-tree.jade index 1093a2548a..420b4bc121 100644 --- a/services/web/app/views/project/editor/file-tree.jade +++ b/services/web/app/views/project/editor/file-tree.jade @@ -115,14 +115,15 @@ script(type='text/ng-template', id='entityListItemTemplate') span( ng-hide="entity.renaming" ) {{ entity.name }} - input( - ng-if="permissions.write", - ng-show="entity.renaming", - ng-model="inputs.name", - ng-blur="finishRenaming()", - select-name-when="entity.renaming", - on-enter="finishRenaming()" - ) + span.rename-input + input( + ng-if="permissions.write", + ng-show="entity.renaming", + ng-model="inputs.name", + ng-blur="finishRenaming()", + select-name-when="entity.renaming", + on-enter="finishRenaming()" + ) span.dropdown( ng-show="entity.selected", @@ -197,14 +198,15 @@ script(type='text/ng-template', id='entityListItemTemplate') span( ng-hide="entity.renaming" ) {{ entity.name }} - input( - ng-if="permissions.write", - ng-show="entity.renaming", - ng-model="inputs.name", - ng-blur="finishRenaming()", - select-name-when="entity.renaming", - on-enter="finishRenaming()" - ) + span.rename-input + input( + ng-if="permissions.write", + ng-show="entity.renaming", + ng-model="inputs.name", + ng-blur="finishRenaming()", + select-name-when="entity.renaming", + on-enter="finishRenaming()" + ) span.dropdown( ng-if="permissions.write" diff --git a/services/web/public/stylesheets/app/editor/file-tree.less b/services/web/public/stylesheets/app/editor/file-tree.less index 2000d85dce..7e2a3117fa 100644 --- a/services/web/public/stylesheets/app/editor/file-tree.less +++ b/services/web/public/stylesheets/app/editor/file-tree.less @@ -31,6 +31,7 @@ aside#file-tree { li { line-height: 2.6; + position: relative; .entity-name { color: @gray-darker; @@ -85,6 +86,17 @@ aside#file-tree { padding: 0 12px; } } + + .rename-input { + display: block; + position: absolute; + top: 1px; + left: 44px; + right: 32px; + input { + width: 100%; + } + } } } From 6bfefea5cb9024ff6c2fbad328273662f40bd156 Mon Sep 17 00:00:00 2001 From: James Allen Date: Tue, 12 Aug 2014 13:18:22 +0100 Subject: [PATCH 07/12] Show PDF button in toolbar if file tree is closed in PDF flat view --- .../web/app/views/project/editor/file-tree.jade | 5 +++-- .../web/app/views/project/editor/header.jade | 11 +++++++++++ .../web/public/coffee/ide/pdf/PdfManager.coffee | 1 + .../controllers/PdfViewToggleController.coffee | 17 +++++++++++++++++ .../stylesheets/app/editor/file-tree.less | 2 +- .../web/public/stylesheets/app/editor/pdf.less | 14 -------------- .../public/stylesheets/app/editor/toolbar.less | 16 ++++++++++++++-- 7 files changed, 47 insertions(+), 19 deletions(-) create mode 100644 services/web/public/coffee/ide/pdf/controllers/PdfViewToggleController.coffee diff --git a/services/web/app/views/project/editor/file-tree.jade b/services/web/app/views/project/editor/file-tree.jade index 420b4bc121..f5f4b8a9d6 100644 --- a/services/web/app/views/project/editor/file-tree.jade +++ b/services/web/app/views/project/editor/file-tree.jade @@ -46,14 +46,15 @@ aside#file-tree(ng-controller="FileTreeController").full-size ng-class="{ 'no-toolbar': !permissions.write }" ) - div(ng-show="ui.pdfLayout == 'flat' && (ui.view == 'editor' || ui.view == 'pdf')") + div(ng-show="ui.pdfLayout == 'flat' && (ui.view == 'editor' || ui.view == 'pdf' || ui.view == 'file')") ul.list-unstyled.file-tree-list li( ng-class="{ 'selected': ui.view == 'pdf' }" + ng-controller="PdfViewToggleController" ) .entity .entity-name( - ng-click="ui.view = 'pdf'" + ng-click="togglePdfView()" ) i.fa.fa-fw.toggle i.fa.fa-fw.fa-file-pdf-o diff --git a/services/web/app/views/project/editor/header.jade b/services/web/app/views/project/editor/header.jade index a119c561db..865432b7b9 100644 --- a/services/web/app/views/project/editor/header.jade +++ b/services/web/app/views/project/editor/header.jade @@ -14,6 +14,17 @@ header.toolbar.toolbar-header(ng-cloak, ng-hide="state.loading") tooltip-append-to-body="true" ) i.fa.fa-fw.fa-level-up + span(ng-controller="PdfViewToggleController") + a( + href, + ng-show="ui.pdfLayout == 'flat' && fileTreeClosed", + tooltip="PDF", + tooltip-placement="bottom", + tooltip-append-to-body="true", + ng-click="togglePdfView()", + ng-class="{ 'active': ui.view == 'pdf' }" + ) + i.fa.fa-file-pdf-o .toolbar-center.project-name(ng-controller="ProjectNameController") span.name( diff --git a/services/web/public/coffee/ide/pdf/PdfManager.coffee b/services/web/public/coffee/ide/pdf/PdfManager.coffee index 47ac42a1b6..f53d649e97 100644 --- a/services/web/public/coffee/ide/pdf/PdfManager.coffee +++ b/services/web/public/coffee/ide/pdf/PdfManager.coffee @@ -1,5 +1,6 @@ define [ "ide/pdf/controllers/PdfController" + "ide/pdf/controllers/PdfViewToggleController" "ide/pdf/directives/pdfJs" ], () -> class PdfManager diff --git a/services/web/public/coffee/ide/pdf/controllers/PdfViewToggleController.coffee b/services/web/public/coffee/ide/pdf/controllers/PdfViewToggleController.coffee new file mode 100644 index 0000000000..156d46dd3e --- /dev/null +++ b/services/web/public/coffee/ide/pdf/controllers/PdfViewToggleController.coffee @@ -0,0 +1,17 @@ +define [ + "base" +], (App) -> + App.controller "PdfViewToggleController", ($scope) -> + $scope.togglePdfView = () -> + if $scope.ui.view == "pdf" + $scope.ui.view = "editor" + else + $scope.ui.view = "pdf" + + $scope.fileTreeClosed = false + $scope.$on "layout:main:resize", (e, state) -> + if state.west.initClosed + $scope.fileTreeClosed = true + else + $scope.fileTreeClosed = false + $scope.$apply() \ No newline at end of file diff --git a/services/web/public/stylesheets/app/editor/file-tree.less b/services/web/public/stylesheets/app/editor/file-tree.less index 7e2a3117fa..0356c873f3 100644 --- a/services/web/public/stylesheets/app/editor/file-tree.less +++ b/services/web/public/stylesheets/app/editor/file-tree.less @@ -72,7 +72,7 @@ aside#file-tree { color: @link-color; border-right: 4px solid @link-color; font-weight: bold; - i.fa-folder-open, i.fa-folder, i.fa-file, i.fa-image { + i.fa-folder-open, i.fa-folder, i.fa-file, i.fa-image, i.fa-file-pdf-o { color: @link-color; } padding-right: 32px; diff --git a/services/web/public/stylesheets/app/editor/pdf.less b/services/web/public/stylesheets/app/editor/pdf.less index 17c176305a..8b7bacb047 100644 --- a/services/web/public/stylesheets/app/editor/pdf.less +++ b/services/web/public/stylesheets/app/editor/pdf.less @@ -74,20 +74,6 @@ } .pdf .toolbar { - .log-btn { - &.active, &:active { - .label { - display: none; - } - color: white; - background-color: @link-color; - .box-shadow(inset 0 3px 5px rgba(0, 0, 0, 0.225)); - &:hover { - color: white; - } - } - } - .toolbar-right { margin-right: @line-height-computed / 2; a { diff --git a/services/web/public/stylesheets/app/editor/toolbar.less b/services/web/public/stylesheets/app/editor/toolbar.less index 2b63b41a10..30bc95aaaa 100644 --- a/services/web/public/stylesheets/app/editor/toolbar.less +++ b/services/web/public/stylesheets/app/editor/toolbar.less @@ -16,14 +16,25 @@ a:not(.btn) { display: inline-block; color: @gray-light; - padding: 5px 12px 6px; - margin: 0; + padding: 4px 10px 5px; + margin: 1px 2px; border-radius: @border-radius-small; &:hover { text-shadow: 0 1px 0 rgba(0, 0, 0, 0.25); color: @gray-dark; text-decoration: none; } + &.active, &:active { + .label { + display: none; + } + color: white; + background-color: @link-color; + .box-shadow(inset 0 3px 5px rgba(0, 0, 0, 0.225)); + &:hover { + color: white; + } + } } .btn-full-height { @@ -81,6 +92,7 @@ height: 32px; a { padding: 4px 2px 2px; + margin: 0; margin-left: 6px; } .toolbar-right { From 2a6041752d12b11e0b587d0e3a9ab5fabc3b7359 Mon Sep 17 00:00:00 2001 From: James Allen Date: Wed, 13 Aug 2014 15:44:30 +0100 Subject: [PATCH 08/12] Remove unescaped strings on bonus page --- services/web/app/views/referal/bonus.jade | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/services/web/app/views/referal/bonus.jade b/services/web/app/views/referal/bonus.jade index 3439e4694e..22e72ff6f0 100644 --- a/services/web/app/views/referal/bonus.jade +++ b/services/web/app/views/referal/bonus.jade @@ -80,11 +80,11 @@ block content .row.ab-bonus .col-md-10.col-md-offset-1.bonus-banner - if (refered_user_count == 0) - p.thanks #{translate("you_not_introed_anyone_to_sl")} + p.thanks !{translate("you_not_introed_anyone_to_sl")} - else if (refered_user_count == 1) - p.thanks #{translate("you_introed_small_number", {numberOfPeople:"#{refered_user_count}"})} + p.thanks !{translate("you_introed_small_number", {numberOfPeople:"" + refered_user_count + ""})} - else - p.thanks #{translate("you_introed_high_number", {numberOfPeople:"#{refered_user_count}"})} + p.thanks !{translate("you_introed_high_number", {numberOfPeople:"" + refered_user_count + ""})} script(type="text/ng-template", id="BonusLinkToUsModal") .modal-header From 670e8e5cb90c520bddceeff11eb72843faf83cb4 Mon Sep 17 00:00:00 2001 From: James Allen Date: Wed, 13 Aug 2014 17:26:18 +0100 Subject: [PATCH 09/12] Use Dropbox Real-time polling --- .../Features/Dropbox/DropboxRouter.coffee | 15 ++++++ .../Dropbox/DropboxWebhookController.coffee | 15 ++++++ .../Dropbox/DropboxWebhookHandler.coffee | 25 +++++++++ .../ServerAdmin/AdminController.coffee | 4 -- .../TpdsUpdateSender.coffee | 11 +++- services/web/app/coffee/router.coffee | 10 ++-- services/web/app/views/admin.jade | 5 -- .../web/app/views/project/editor/dropbox.jade | 4 -- .../controllers/DropboxController.coffee | 12 +---- .../DropboxWebhookControllerTests.coffee | 48 +++++++++++++++++ .../Dropbox/DropboxWebhookHandlerTests.coffee | 51 +++++++++++++++++++ .../TpdsUpdateSenderTests.coffee | 17 +++++++ 12 files changed, 185 insertions(+), 32 deletions(-) create mode 100644 services/web/app/coffee/Features/Dropbox/DropboxRouter.coffee create mode 100644 services/web/app/coffee/Features/Dropbox/DropboxWebhookController.coffee create mode 100644 services/web/app/coffee/Features/Dropbox/DropboxWebhookHandler.coffee create mode 100644 services/web/test/UnitTests/coffee/Dropbox/DropboxWebhookControllerTests.coffee create mode 100644 services/web/test/UnitTests/coffee/Dropbox/DropboxWebhookHandlerTests.coffee diff --git a/services/web/app/coffee/Features/Dropbox/DropboxRouter.coffee b/services/web/app/coffee/Features/Dropbox/DropboxRouter.coffee new file mode 100644 index 0000000000..68d8365d5e --- /dev/null +++ b/services/web/app/coffee/Features/Dropbox/DropboxRouter.coffee @@ -0,0 +1,15 @@ +DropboxUserController = require './DropboxUserController' +DropboxWebhookController = require './DropboxWebhookController' + +module.exports = + apply: (app) -> + app.get '/dropbox/beginAuth', DropboxUserController.redirectUserToDropboxAuth + app.get '/dropbox/completeRegistration', DropboxUserController.completeDropboxRegistration + app.get '/dropbox/unlink', DropboxUserController.unlinkDropbox + + app.get '/dropbox/webhook', DropboxWebhookController.verify + app.post '/dropbox/webhook', DropboxWebhookController.webhook + app.ignoreCsrf('post', '/dropbox/webhook') + + + diff --git a/services/web/app/coffee/Features/Dropbox/DropboxWebhookController.coffee b/services/web/app/coffee/Features/Dropbox/DropboxWebhookController.coffee new file mode 100644 index 0000000000..3509d58687 --- /dev/null +++ b/services/web/app/coffee/Features/Dropbox/DropboxWebhookController.coffee @@ -0,0 +1,15 @@ +logger = require("logger-sharelatex") +DropboxWebhookHandler = require("./DropboxWebhookHandler") + +module.exports = DropboxWebhookController = + verify: (req, res, next = (error) ->) -> + res.send(req.query.challenge) + + webhook: (req, res, next = (error) ->) -> + dropbox_uids = req.body?.delta?.users + logger.log dropbox_uids: dropbox_uids, "received webhook request from Dropbox" + if !dropbox_uids? + return res.send(400) # Bad Request + DropboxWebhookHandler.pollDropboxUids dropbox_uids, (error) -> + return next(error) if error? + res.send(200) \ No newline at end of file diff --git a/services/web/app/coffee/Features/Dropbox/DropboxWebhookHandler.coffee b/services/web/app/coffee/Features/Dropbox/DropboxWebhookHandler.coffee new file mode 100644 index 0000000000..2b27ced163 --- /dev/null +++ b/services/web/app/coffee/Features/Dropbox/DropboxWebhookHandler.coffee @@ -0,0 +1,25 @@ +logger = require("logger-sharelatex") +async = require "async" +User = require("../../models/User").User +TpdsUpdateSender = require "../ThirdPartyDataStore/TpdsUpdateSender" + +module.exports = DropboxWebhookHandler = + pollDropboxUids: (dropbox_uids, callback = (error) ->) -> + jobs = [] + for uid in dropbox_uids + do (uid) -> + jobs.push (callback) -> + DropboxWebhookHandler.pollDropboxUid uid, callback + async.series jobs, callback + + pollDropboxUid: (dropbox_uid, callback = (error) ->) -> + User.find { + "dropbox.access_token.uid": dropbox_uid + "features.dropbox": true + }, (error, users = []) -> + return callback(error) if error? + user = users[0] + if !user? + logger.log dropbox_uid: dropbox_uid, "no sharelatex user found" + return callback() + TpdsUpdateSender.pollDropboxForUser user._id, callback \ No newline at end of file diff --git a/services/web/app/coffee/Features/ServerAdmin/AdminController.coffee b/services/web/app/coffee/Features/ServerAdmin/AdminController.coffee index a8f92a5fe8..8a1a22e00b 100755 --- a/services/web/app/coffee/Features/ServerAdmin/AdminController.coffee +++ b/services/web/app/coffee/Features/ServerAdmin/AdminController.coffee @@ -73,10 +73,6 @@ module.exports = AdminController = flushProjectToTpds: (req, res)-> projectEntityHandler.flushProjectToThirdPartyDataStore req.body.project_id, (err)-> res.send 200 - - pollUsersWithDropbox: (req, res)-> - TpdsPollingBackgroundTasks.pollUsersWithDropbox -> - res.send 200 createMessage: (req, res, next) -> SystemMessageManager.createMessage req.body.content, (error) -> diff --git a/services/web/app/coffee/Features/ThirdPartyDataStore/TpdsUpdateSender.coffee b/services/web/app/coffee/Features/ThirdPartyDataStore/TpdsUpdateSender.coffee index ccaddc22a1..b341e7b574 100644 --- a/services/web/app/coffee/Features/ThirdPartyDataStore/TpdsUpdateSender.coffee +++ b/services/web/app/coffee/Features/ThirdPartyDataStore/TpdsUpdateSender.coffee @@ -87,7 +87,16 @@ module.exports = title:"deleteEntity" sl_all_user_ids:JSON.stringify(allUserIds) queue.enqueue options.project_id, "standardHttpRequest", deleteOptions, callback - + + pollDropboxForUser: (user_id, callback = (err) ->) -> + metrics.inc("tpds.poll-dropbox") + logger.log user_id: user_id, "polling dropbox for user" + options = + method: "POST" + uri:"#{settings.apis.thirdPartyDataStore.url}/user/poll" + json: + user_ids: [user_id] + queue.enqueue "poll-dropbox:#{user_id}", "standardHttpRequest", options, callback getProjectsUsersIds = (project_id, callback = (err, owner_id, allUserIds)->)-> Project.findById project_id, "_id owner_ref readOnly_refs collaberator_refs", (err, project)-> diff --git a/services/web/app/coffee/router.coffee b/services/web/app/coffee/router.coffee index 0fbb31e18c..198b58c4cb 100644 --- a/services/web/app/coffee/router.coffee +++ b/services/web/app/coffee/router.coffee @@ -10,7 +10,6 @@ EditorHttpController = require("./Features/Editor/EditorHttpController") EditorUpdatesController = require("./Features/Editor/EditorUpdatesController") Settings = require('settings-sharelatex') TpdsController = require('./Features/ThirdPartyDataStore/TpdsController') -dropboxHandler = require('./Features/Dropbox/DropboxHandler') SubscriptionRouter = require './Features/Subscription/SubscriptionRouter' UploadsRouter = require './Features/Uploads/UploadsRouter' metrics = require('./infrastructure/Metrics') @@ -32,13 +31,14 @@ HealthCheckController = require("./Features/HealthCheck/HealthCheckController") ProjectDownloadsController = require "./Features/Downloads/ProjectDownloadsController" FileStoreController = require("./Features/FileStore/FileStoreController") TrackChangesController = require("./Features/TrackChanges/TrackChangesController") -DropboxUserController = require("./Features/Dropbox/DropboxUserController") PasswordResetRouter = require("./Features/PasswordReset/PasswordResetRouter") StaticPagesRouter = require("./Features/StaticPages/StaticPagesRouter") ChatController = require("./Features/Chat/ChatController") BlogController = require("./Features/Blog/BlogController") WikiController = require("./Features/Wiki/WikiController") ConnectedUsersController = require("./Features/ConnectedUsers/ConnectedUsersController") +DropboxRouter = require "./Features/Dropbox/DropboxRouter" +dropboxHandler = require "./Features/Dropbox/DropboxHandler" logger = require("logger-sharelatex") _ = require("underscore") @@ -66,6 +66,7 @@ module.exports = class Router PasswordResetRouter.apply(app) StaticPagesRouter.apply(app) TemplatesRouter.apply(app) + DropboxRouter.apply(app) app.get '/blog', BlogController.getIndexPage app.get '/blog/*', BlogController.getPage @@ -80,10 +81,6 @@ module.exports = class Router app.del '/user/newsletter/unsubscribe', AuthenticationController.requireLogin(), UserController.unsubscribe app.del '/user', AuthenticationController.requireLogin(), UserController.deleteUser - app.get '/dropbox/beginAuth', DropboxUserController.redirectUserToDropboxAuth - app.get '/dropbox/completeRegistration', DropboxUserController.completeDropboxRegistration - app.get '/dropbox/unlink', DropboxUserController.unlinkDropbox - app.get '/user/auth_token', AuthenticationController.requireLogin(), AuthenticationController.getAuthToken app.get '/user/personal_info', AuthenticationController.requireLogin(allow_auth_token: true), UserInfoController.getLoggedInUsersPersonalInfo app.get '/user/:user_id/personal_info', httpAuth, UserInfoController.getPersonalInfo @@ -172,7 +169,6 @@ module.exports = class Router app.post '/admin/dissconectAllUsers', SecurityManager.requestIsAdmin, AdminController.dissconectAllUsers app.post '/admin/syncUserToSubscription', SecurityManager.requestIsAdmin, AdminController.syncUserToSubscription app.post '/admin/flushProjectToTpds', SecurityManager.requestIsAdmin, AdminController.flushProjectToTpds - app.post '/admin/pollUsersWithDropbox', SecurityManager.requestIsAdmin, AdminController.pollUsersWithDropbox app.post '/admin/messages', SecurityManager.requestIsAdmin, AdminController.createMessage app.post '/admin/messages/clear', SecurityManager.requestIsAdmin, AdminController.clearMessages diff --git a/services/web/app/views/admin.jade b/services/web/app/views/admin.jade index 953317fd7c..aa5da2e608 100644 --- a/services/web/app/views/admin.jade +++ b/services/web/app/views/admin.jade @@ -55,11 +55,6 @@ block content input.form-control(type='text', name='project_id', placeholder='project_id', required) .form-group button.btn-primary.btn(type='submit') Flush - hr - .row-spaced - form(enctype='multipart/form-data', method='post',action='/admin/pollUsersWithDropbox') - input(name="_csrf", type="hidden", value=csrfToken) - button.btn.btn-primary(type="submit") Poll users with dropbox tab(heading="System Messages") each message in systemMessages diff --git a/services/web/app/views/project/editor/dropbox.jade b/services/web/app/views/project/editor/dropbox.jade index 3d86f57d60..7cc425ccef 100644 --- a/services/web/app/views/project/editor/dropbox.jade +++ b/services/web/app/views/project/editor/dropbox.jade @@ -20,10 +20,6 @@ script(type="text/ng-template", id="dropboxModalTemplate") div(ng-show="dbState.hasDropboxFeature && dbState.userIsLinkedToDropbox") - progressbar.progress-striped.active(value='dbState.percentageLeftTillNextPoll', type="info") - p - strong {{dbState.minsTillNextPoll}} #{translate("minutes")} - span #{translate("until_db_checked_for_changes")} p.small | #{translate("this_project_will_appear_in_your_dropbox_folder_at")} strong Dropbox/sharelatex/{{ project.name }} diff --git a/services/web/public/coffee/ide/dropbox/controllers/DropboxController.coffee b/services/web/public/coffee/ide/dropbox/controllers/DropboxController.coffee index afe4be5c4c..f5186f67f5 100644 --- a/services/web/public/coffee/ide/dropbox/controllers/DropboxController.coffee +++ b/services/web/public/coffee/ide/dropbox/controllers/DropboxController.coffee @@ -27,20 +27,10 @@ define [ $scope.dbState = cachedState $scope.dbState.hasDropboxFeature = $scope.project.features.dropbox - - calculatePollTime = -> - ide.socket.emit "getLastTimePollHappned", (err, lastTimePollHappened)=> - milisecondsSinceLastPoll = new Date().getTime() - lastTimePollHappened - roundedMinsSinceLastPoll = Math.round(milisecondsSinceLastPoll / ONE_MIN_MILI) - - $scope.dbState.minsTillNextPoll = POLLING_INTERVAL - roundedMinsSinceLastPoll - $scope.dbState.percentageLeftTillNextPoll = ((roundedMinsSinceLastPoll / POLLING_INTERVAL) * 100) - $timeout calculatePollTime, 60 * 1000 ide.socket.emit "getUserDropboxLinkStatus", user_id, (err, status)=> $scope.dbState.gotLinkStatus = true - if status.registered - calculatePollTime() + if status.registered $scope.dbState.userIsLinkedToDropbox = true cachedState = $scope.dbState diff --git a/services/web/test/UnitTests/coffee/Dropbox/DropboxWebhookControllerTests.coffee b/services/web/test/UnitTests/coffee/Dropbox/DropboxWebhookControllerTests.coffee new file mode 100644 index 0000000000..ad72827c78 --- /dev/null +++ b/services/web/test/UnitTests/coffee/Dropbox/DropboxWebhookControllerTests.coffee @@ -0,0 +1,48 @@ +SandboxedModule = require('sandboxed-module') +assert = require('assert') +require('chai').should() +sinon = require('sinon') +modulePath = require('path').join __dirname, '../../../../app/js/Features/Dropbox/DropboxWebhookController.js' + +describe 'DropboxWebhookController', -> + beforeEach -> + + @DropboxWebhookController = SandboxedModule.require modulePath, requires: + "./DropboxWebhookHandler": @DropboxWebhookHandler = {} + 'logger-sharelatex': + log:-> + err:-> + + describe "verify", -> + beforeEach -> + @res = + send: sinon.stub() + @req.query = + challenge: @challenge = "foo" + @DropboxWebhookController.verify(@req, @res) + + it "should echo the challenge parameter back", -> + @res.send.calledWith(@challenge).should.equal true + + describe "webhook", -> + beforeEach -> + @req.body = + delta: + users: @dropbox_uids = [ + "123456", + "789123" + ] + @res.send = sinon.stub() + @DropboxWebhookHandler.pollDropboxUids = sinon.stub().callsArg(1) + @DropboxWebhookController.webhook(@req, @res) + + it "should poll the Dropbox uids", -> + @DropboxWebhookHandler.pollDropboxUids + .calledWith(@dropbox_uids) + .should.equal true + + it "should return success", -> + @res.send + .calledWith(200) + .should.equal true + diff --git a/services/web/test/UnitTests/coffee/Dropbox/DropboxWebhookHandlerTests.coffee b/services/web/test/UnitTests/coffee/Dropbox/DropboxWebhookHandlerTests.coffee new file mode 100644 index 0000000000..82a0220858 --- /dev/null +++ b/services/web/test/UnitTests/coffee/Dropbox/DropboxWebhookHandlerTests.coffee @@ -0,0 +1,51 @@ +SandboxedModule = require('sandboxed-module') +assert = require('assert') +require('chai').should() +sinon = require('sinon') +modulePath = require('path').join __dirname, '../../../../app/js/Features/Dropbox/DropboxWebhookHandler.js' + +describe 'DropboxWebhookHandler', -> + beforeEach -> + @DropboxWebhookHandler = SandboxedModule.require modulePath, requires: + "../../models/User": User: @User = {} + "../ThirdPartyDataStore/TpdsUpdateSender": @TpdsUpdateSender = {} + 'logger-sharelatex': + log:-> + err:-> + @callback = sinon.stub() + + describe "pollDropboxUids", -> + beforeEach (done) -> + @dropbox_uids = [ + "123456", + "789123" + ] + @DropboxWebhookHandler.pollDropboxUid = sinon.stub().callsArg(1) + @DropboxWebhookHandler.pollDropboxUids @dropbox_uids, done + + it "should call pollDropboxUid for each uid", -> + for uid in @dropbox_uids + @DropboxWebhookHandler.pollDropboxUid + .calledWith(uid) + .should.equal true + + describe "pollDropboxUid", -> + beforeEach -> + @dropbox_uid = "dropbox-123456" + @user_id = "sharelatex-user-id" + @User.find = sinon.stub().callsArgWith(1, null, [ _id: @user_id ]) + @TpdsUpdateSender.pollDropboxForUser = sinon.stub().callsArg(1) + @DropboxWebhookHandler.pollDropboxUid @dropbox_uid, @callback + + it "should look up the user", -> + @User.find + .calledWith({ "dropbox.access_token.uid": @dropbox_uid, "features.dropbox": true }) + .should.equal true + + it "should poll the user's Dropbox", -> + @TpdsUpdateSender.pollDropboxForUser + .calledWith(@user_id) + .should.equal true + + it "should call the callback", -> + @callback.called.should.equal true diff --git a/services/web/test/UnitTests/coffee/ThirdPartyDataStore/TpdsUpdateSenderTests.coffee b/services/web/test/UnitTests/coffee/ThirdPartyDataStore/TpdsUpdateSenderTests.coffee index 28dcce5571..2b7b691d8b 100644 --- a/services/web/test/UnitTests/coffee/ThirdPartyDataStore/TpdsUpdateSenderTests.coffee +++ b/services/web/test/UnitTests/coffee/ThirdPartyDataStore/TpdsUpdateSenderTests.coffee @@ -105,3 +105,20 @@ describe 'TpdsUpdateSender', -> job.headers.sl_all_user_ids.should.eql(JSON.stringify([collaberator_ref_1, read_only_ref_1, user_id])) done() @updateSender.moveEntity {project_id:project_id, project_name:oldProjectName, newProjectName:newProjectName} + + it "pollDropboxForUser", (done) -> + @requestQueuer.enqueue = sinon.stub().callsArg(3) + @updateSender.pollDropboxForUser user_id, (error) => + @requestQueuer.enqueue + .calledWith( + "poll-dropbox:#{user_id}", + "standardHttpRequest", + { + method: "POST" + uri: "#{thirdPartyDataStoreApiUrl}/user/poll" + json: + user_ids: [user_id] + } + ) + .should.equal true + done() From 604505131bd122ea938727e14ce15202324ba79c Mon Sep 17 00:00:00 2001 From: James Allen Date: Wed, 13 Aug 2014 17:34:45 +0100 Subject: [PATCH 10/12] Convert integer dropbox id to string when searching mongo --- .../app/coffee/Features/Dropbox/DropboxWebhookHandler.coffee | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/services/web/app/coffee/Features/Dropbox/DropboxWebhookHandler.coffee b/services/web/app/coffee/Features/Dropbox/DropboxWebhookHandler.coffee index 2b27ced163..8e48489bed 100644 --- a/services/web/app/coffee/Features/Dropbox/DropboxWebhookHandler.coffee +++ b/services/web/app/coffee/Features/Dropbox/DropboxWebhookHandler.coffee @@ -14,7 +14,7 @@ module.exports = DropboxWebhookHandler = pollDropboxUid: (dropbox_uid, callback = (error) ->) -> User.find { - "dropbox.access_token.uid": dropbox_uid + "dropbox.access_token.uid": dropbox_uid.toString() "features.dropbox": true }, (error, users = []) -> return callback(error) if error? From 7816810249d9d35d323adaa1e55556faed2d84a8 Mon Sep 17 00:00:00 2001 From: James Allen Date: Thu, 14 Aug 2014 10:12:01 +0100 Subject: [PATCH 11/12] Add in poll dropbox endpoint back into admin --- .../Features/ServerAdmin/AdminController.coffee | 7 ++++++- services/web/app/coffee/router.coffee | 1 + services/web/app/views/admin.jade | 12 +++++++++++- 3 files changed, 18 insertions(+), 2 deletions(-) diff --git a/services/web/app/coffee/Features/ServerAdmin/AdminController.coffee b/services/web/app/coffee/Features/ServerAdmin/AdminController.coffee index 8a1a22e00b..06c566f0aa 100755 --- a/services/web/app/coffee/Features/ServerAdmin/AdminController.coffee +++ b/services/web/app/coffee/Features/ServerAdmin/AdminController.coffee @@ -12,7 +12,7 @@ rclient.auth(Settings.redis.web.password) RecurlyWrapper = require('../Subscription/RecurlyWrapper') SubscriptionHandler = require('../Subscription/SubscriptionHandler') projectEntityHandler = require('../Project/ProjectEntityHandler') -TpdsPollingBackgroundTasks = require("../ThirdPartyDataStore/TpdsPollingBackgroundTasks") +TpdsUpdateSender = require("../ThirdPartyDataStore/TpdsUpdateSender") EditorRealTimeController = require("../Editor/EditorRealTimeController") SystemMessageManager = require("../SystemMessages/SystemMessageManager") @@ -73,6 +73,11 @@ module.exports = AdminController = flushProjectToTpds: (req, res)-> projectEntityHandler.flushProjectToThirdPartyDataStore req.body.project_id, (err)-> res.send 200 + + pollDropboxForUser: (req, res)-> + user_id = req.body.user_id + TpdsUpdateSender.pollDropboxForUser user_id, () -> + res.send 200 createMessage: (req, res, next) -> SystemMessageManager.createMessage req.body.content, (error) -> diff --git a/services/web/app/coffee/router.coffee b/services/web/app/coffee/router.coffee index 198b58c4cb..6379b96f61 100644 --- a/services/web/app/coffee/router.coffee +++ b/services/web/app/coffee/router.coffee @@ -169,6 +169,7 @@ module.exports = class Router app.post '/admin/dissconectAllUsers', SecurityManager.requestIsAdmin, AdminController.dissconectAllUsers app.post '/admin/syncUserToSubscription', SecurityManager.requestIsAdmin, AdminController.syncUserToSubscription app.post '/admin/flushProjectToTpds', SecurityManager.requestIsAdmin, AdminController.flushProjectToTpds + app.post '/admin/pollDropboxForUser', SecurityManager.requestIsAdmin, AdminController.pollDropboxForUser app.post '/admin/messages', SecurityManager.requestIsAdmin, AdminController.createMessage app.post '/admin/messages/clear', SecurityManager.requestIsAdmin, AdminController.clearMessages diff --git a/services/web/app/views/admin.jade b/services/web/app/views/admin.jade index aa5da2e608..8b03b60253 100644 --- a/services/web/app/views/admin.jade +++ b/services/web/app/views/admin.jade @@ -45,7 +45,7 @@ block content .form-group button.btn-primary.btn(type='submit') Link - tab(heading="TPDS Management") + tab(heading="TPDS/Dropbox Management") h3 Flush project to TPDS .row form.col-xs-6(enctype='multipart/form-data', method='post',action='/admin/flushProjectToTpds') @@ -55,6 +55,16 @@ block content input.form-control(type='text', name='project_id', placeholder='project_id', required) .form-group button.btn-primary.btn(type='submit') Flush + hr + h3 Poll Dropbox for user + .row + form.col-xs-6(enctype='multipart/form-data', method='post',action='/admin/pollDropboxForUser') + input(name="_csrf", type="hidden", value=csrfToken) + .form-group + label(for='user_id') user_id + input.form-control(type='text', name='user_id', placeholder='user_id', required) + .form-group + button.btn-primary.btn(type='submit') Poll tab(heading="System Messages") each message in systemMessages From 9be72282478fd7ba89584f99ffc7b60c4db2e8cf Mon Sep 17 00:00:00 2001 From: James Allen Date: Thu, 14 Aug 2014 13:48:23 +0100 Subject: [PATCH 12/12] Delay Dropbox polling by 5 seconds and dispatch only a single request --- .../Dropbox/DropboxWebhookController.coffee | 8 ++- .../Dropbox/DropboxWebhookHandler.coffee | 42 ++++++++--- .../Dropbox/DropboxWebhookHandlerTests.coffee | 70 ++++++++++++++++--- 3 files changed, 98 insertions(+), 22 deletions(-) diff --git a/services/web/app/coffee/Features/Dropbox/DropboxWebhookController.coffee b/services/web/app/coffee/Features/Dropbox/DropboxWebhookController.coffee index 3509d58687..5726b51bc4 100644 --- a/services/web/app/coffee/Features/Dropbox/DropboxWebhookController.coffee +++ b/services/web/app/coffee/Features/Dropbox/DropboxWebhookController.coffee @@ -10,6 +10,10 @@ module.exports = DropboxWebhookController = logger.log dropbox_uids: dropbox_uids, "received webhook request from Dropbox" if !dropbox_uids? return res.send(400) # Bad Request + + # Do this in the background so as not to keep Dropbox waiting DropboxWebhookHandler.pollDropboxUids dropbox_uids, (error) -> - return next(error) if error? - res.send(200) \ No newline at end of file + if error? + logger.error err: error, dropbox_uids: dropbox_uids, "error in webhook" + + res.send(200) \ No newline at end of file diff --git a/services/web/app/coffee/Features/Dropbox/DropboxWebhookHandler.coffee b/services/web/app/coffee/Features/Dropbox/DropboxWebhookHandler.coffee index 8e48489bed..d29a0d3aa5 100644 --- a/services/web/app/coffee/Features/Dropbox/DropboxWebhookHandler.coffee +++ b/services/web/app/coffee/Features/Dropbox/DropboxWebhookHandler.coffee @@ -1,8 +1,13 @@ logger = require("logger-sharelatex") +settings = require("settings-sharelatex") async = require "async" User = require("../../models/User").User TpdsUpdateSender = require "../ThirdPartyDataStore/TpdsUpdateSender" +redis = require('redis') +rclient = redis.createClient(settings.redis.web.port, settings.redis.web.host) +rclient.auth(settings.redis.web.password) + module.exports = DropboxWebhookHandler = pollDropboxUids: (dropbox_uids, callback = (error) ->) -> jobs = [] @@ -13,13 +18,32 @@ module.exports = DropboxWebhookHandler = async.series jobs, callback pollDropboxUid: (dropbox_uid, callback = (error) ->) -> - User.find { - "dropbox.access_token.uid": dropbox_uid.toString() - "features.dropbox": true - }, (error, users = []) -> + DropboxWebhookHandler._delayAndBatchPoll dropbox_uid, (error, shouldPoll) -> return callback(error) if error? - user = users[0] - if !user? - logger.log dropbox_uid: dropbox_uid, "no sharelatex user found" - return callback() - TpdsUpdateSender.pollDropboxForUser user._id, callback \ No newline at end of file + return callback() if !shouldPoll + User.find { + "dropbox.access_token.uid": dropbox_uid.toString() + "features.dropbox": true + }, (error, users = []) -> + return callback(error) if error? + user = users[0] + if !user? + logger.log dropbox_uid: dropbox_uid, "no sharelatex user found" + return callback() + TpdsUpdateSender.pollDropboxForUser user._id, callback + + POLL_DELAY_IN_MS: 5000 # 5 seconds + _delayAndBatchPoll: (dropbox_uid, callback = (error, shouldPoll) ->) -> + rclient.set( + "dropbox-poll-lock:#{dropbox_uid}", "LOCK", + "PX", DropboxWebhookHandler.POLL_DELAY_IN_MS, + "NX", + (error, gotLock) -> + return callback(error) if error? + if gotLock + setTimeout () -> + callback(null, true) + , DropboxWebhookHandler.POLL_DELAY_IN_MS + else + callback(null, false) + ) \ No newline at end of file diff --git a/services/web/test/UnitTests/coffee/Dropbox/DropboxWebhookHandlerTests.coffee b/services/web/test/UnitTests/coffee/Dropbox/DropboxWebhookHandlerTests.coffee index 82a0220858..60519532e2 100644 --- a/services/web/test/UnitTests/coffee/Dropbox/DropboxWebhookHandlerTests.coffee +++ b/services/web/test/UnitTests/coffee/Dropbox/DropboxWebhookHandlerTests.coffee @@ -1,6 +1,7 @@ SandboxedModule = require('sandboxed-module') assert = require('assert') require('chai').should() +expect = require("chai").expect sinon = require('sinon') modulePath = require('path').join __dirname, '../../../../app/js/Features/Dropbox/DropboxWebhookHandler.js' @@ -9,6 +10,10 @@ describe 'DropboxWebhookHandler', -> @DropboxWebhookHandler = SandboxedModule.require modulePath, requires: "../../models/User": User: @User = {} "../ThirdPartyDataStore/TpdsUpdateSender": @TpdsUpdateSender = {} + "redis": + createClient: () => @rclient = + auth: sinon.stub() + 'settings-sharelatex': redis: web: {} 'logger-sharelatex': log:-> err:-> @@ -35,17 +40,60 @@ describe 'DropboxWebhookHandler', -> @user_id = "sharelatex-user-id" @User.find = sinon.stub().callsArgWith(1, null, [ _id: @user_id ]) @TpdsUpdateSender.pollDropboxForUser = sinon.stub().callsArg(1) - @DropboxWebhookHandler.pollDropboxUid @dropbox_uid, @callback - it "should look up the user", -> - @User.find - .calledWith({ "dropbox.access_token.uid": @dropbox_uid, "features.dropbox": true }) - .should.equal true + describe "when there is already a poll in progress", () -> + beforeEach -> + @DropboxWebhookHandler._delayAndBatchPoll = sinon.stub().callsArgWith(1, null, false) + @DropboxWebhookHandler.pollDropboxUid @dropbox_uid, @callback - it "should poll the user's Dropbox", -> - @TpdsUpdateSender.pollDropboxForUser - .calledWith(@user_id) - .should.equal true + it "should not go ahead with the poll", -> + @TpdsUpdateSender.pollDropboxForUser.called.should.equal false + + describe "when we are the one to do the delayed poll", () -> + beforeEach -> + @DropboxWebhookHandler._delayAndBatchPoll = sinon.stub().callsArgWith(1, null, true) + @DropboxWebhookHandler.pollDropboxUid @dropbox_uid, @callback + + it "should look up the user", -> + @User.find + .calledWith({ "dropbox.access_token.uid": @dropbox_uid, "features.dropbox": true }) + .should.equal true + + it "should poll the user's Dropbox", -> + @TpdsUpdateSender.pollDropboxForUser + .calledWith(@user_id) + .should.equal true + + it "should call the callback", -> + @callback.called.should.equal true + + describe "_delayAndBatchPoll", () -> + beforeEach -> + @dropbox_uid = "dropbox-uid-123" + @DropboxWebhookHandler.POLL_DELAY_IN_MS = 100 + + describe "when no one else is polling yet", -> + beforeEach (done) -> + @rclient.set = sinon.stub().callsArgWith(5, null, "OK") + @start = Date.now() + @DropboxWebhookHandler._delayAndBatchPoll @dropbox_uid, (error, @shouldPoll) => + @end = Date.now() + done() + + it "should set the lock", -> + @rclient.set + .calledWith("dropbox-poll-lock:#{@dropbox_uid}", "LOCK", "PX", @DropboxWebhookHandler.POLL_DELAY_IN_MS, "NX") + .should.equal true + + it "should return the callback after the delay with shouldPoll=true", -> + @shouldPoll.should.equal true + expect(@end - @start).to.be.at.least(@DropboxWebhookHandler.POLL_DELAY_IN_MS) + + describe "when someone else is already polling", -> + beforeEach -> + @rclient.set = sinon.stub().callsArgWith(5, null, null) + @DropboxWebhookHandler._delayAndBatchPoll @dropbox_uid, @callback + + it "should return the callback immediately with shouldPoll=false", -> + @callback.calledWith(null, false).should.equal true - it "should call the callback", -> - @callback.called.should.equal true