From 2b349039c35d54230148f21b3ff7d635e25e84b0 Mon Sep 17 00:00:00 2001 From: James Allen Date: Wed, 18 Jun 2014 16:37:18 +0100 Subject: [PATCH] Add in backend multiple project downloading --- .../DocumentUpdaterHandler.coffee | 10 ++- .../ProjectDownloadsController.coffee | 15 +++++ .../Downloads/ProjectZipStreamManager.coffee | 29 +++++++++ .../coffee/managers/SecurityManager.coffee | 18 +++++- services/web/app/coffee/router.coffee | 2 +- .../ProjectDownloadsControllerTests.coffee | 47 ++++++++++++++ .../ProjectZipStreamManagerTests.coffee | 61 ++++++++++++++++++- 7 files changed, 176 insertions(+), 6 deletions(-) diff --git a/services/web/app/coffee/Features/DocumentUpdater/DocumentUpdaterHandler.coffee b/services/web/app/coffee/Features/DocumentUpdater/DocumentUpdaterHandler.coffee index ba38137fb0..3d4d49c827 100644 --- a/services/web/app/coffee/Features/DocumentUpdater/DocumentUpdaterHandler.coffee +++ b/services/web/app/coffee/Features/DocumentUpdater/DocumentUpdaterHandler.coffee @@ -13,7 +13,7 @@ rclient.auth(settings.redis.web.password) Project = require("../../models/Project").Project ProjectLocator = require('../../Features/Project/ProjectLocator') -module.exports = +module.exports = DocumentUpdaterHandler = queueChange : (project_id, doc_id, change, sl_req_id, callback = ()->)-> {callback, sl_req_id} = slReqIdHelper.getCallbackAndReqId(callback, sl_req_id) @@ -42,6 +42,14 @@ module.exports = logger.error err: error, project_id: project_id, sl_req_id: sl_req_id, "document updater returned failure status code: #{res.statusCode}" return callback(error) + flushMultipleProjectsToMongo: (project_ids, callback = (error) ->) -> + jobs = [] + for project_id in project_ids + do (project_id) -> + jobs.push (callback) -> + DocumentUpdaterHandler.flushProjectToMongo project_id, callback + async.series jobs, callback + flushProjectToMongoAndDelete: (project_id, sl_req_id, callback = ()->) -> {callback, sl_req_id} = slReqIdHelper.getCallbackAndReqId(callback, sl_req_id) logger.log project_id:project_id, sl_req_id:sl_req_id, "deleting project from document updater" diff --git a/services/web/app/coffee/Features/Downloads/ProjectDownloadsController.coffee b/services/web/app/coffee/Features/Downloads/ProjectDownloadsController.coffee index 4a817e2988..22272600c9 100644 --- a/services/web/app/coffee/Features/Downloads/ProjectDownloadsController.coffee +++ b/services/web/app/coffee/Features/Downloads/ProjectDownloadsController.coffee @@ -22,4 +22,19 @@ module.exports = ProjectDownloadsController = res.contentType('application/zip') stream.pipe(res) + downloadMultipleProjects: (req, res, next) -> + project_ids = req.query.project_ids.split(",") + Metrics.inc "zip-downloads-multiple" + logger.log project_ids: project_ids, "downloading multiple projects" + DocumentUpdaterHandler.flushMultipleProjectsToMongo project_ids, (error) -> + return next(error) if error? + ProjectZipStreamManager.createZipStreamForMultipleProjects project_ids, (error, stream) -> + return next(error) if error? + res.header( + "Content-Disposition", + "attachment; filename=ShareLaTeX Projects (#{project_ids.length} items).zip" + ) + res.contentType('application/zip') + stream.pipe(res) + diff --git a/services/web/app/coffee/Features/Downloads/ProjectZipStreamManager.coffee b/services/web/app/coffee/Features/Downloads/ProjectZipStreamManager.coffee index 91256025f5..ffc527e3d1 100644 --- a/services/web/app/coffee/Features/Downloads/ProjectZipStreamManager.coffee +++ b/services/web/app/coffee/Features/Downloads/ProjectZipStreamManager.coffee @@ -3,8 +3,37 @@ async = require "async" logger = require "logger-sharelatex" ProjectEntityHandler = require "../Project/ProjectEntityHandler" FileStoreHandler = require("../FileStore/FileStoreHandler") +Project = require("../../models/Project").Project module.exports = ProjectZipStreamManager = + createZipStreamForMultipleProjects: (project_ids, callback = (error, stream) ->) -> + # We'll build up a zip file that contains multiple zip files + + archive = archiver("zip") + archive.on "error", (err)-> + logger.err err:err, project_ids:project_ids, "something went wrong building archive of project" + callback null, archive + + logger.log project_ids: project_ids, "creating zip stream of multiple projects" + + jobs = [] + for project_id in project_ids or [] + do (project_id) -> + jobs.push (callback) -> + Project.findById project_id, "name", (error, project) -> + return callback(error) if error? + logger.log project_id: project_id, name: project.name, "appending project to zip stream" + ProjectZipStreamManager.createZipStreamForProject project_id, (error, stream) -> + return callback(error) if error? + archive.append stream, name: "#{project.name}.zip" + stream.on "end", () -> + logger.log project_id: project_id, name: project.name, "zip stream ended" + callback() + + async.series jobs, () -> + logger.log project_ids: project_ids, "finished creating zip stream of multiple projects" + archive.finalize() + createZipStreamForProject: (project_id, callback = (error, stream) ->) -> archive = archiver("zip") # return stream immediately before we start adding things to it diff --git a/services/web/app/coffee/managers/SecurityManager.coffee b/services/web/app/coffee/managers/SecurityManager.coffee index ddb8ec5bac..8c291e5362 100644 --- a/services/web/app/coffee/managers/SecurityManager.coffee +++ b/services/web/app/coffee/managers/SecurityManager.coffee @@ -9,8 +9,9 @@ AuthenticationController = require("../Features/Authentication/AuthenticationCon _ = require('underscore') metrics = require('../infrastructure/Metrics') querystring = require('querystring') +async = require "async" -module.exports = +module.exports = SecurityManager = restricted : (req, res, next)-> if req.session.user? res.render 'user/restricted', @@ -25,6 +26,21 @@ module.exports = else callback null, null + requestCanAccessMultipleProjects: (req, res, next) -> + project_ids = req.query.project_ids?.split(",") + jobs = [] + for project_id in project_ids or [] + do (project_id) -> + jobs.push (callback) -> + # This is a bit hacky - better to have an abstracted method + # that we can pass project_id to, but this whole file needs + # a serious refactor ATM. + req.params.Project_id = project_id + SecurityManager.requestCanAccessProject req, res, (error) -> + delete req.params.Project_id + callback(error) + async.series jobs, next + requestCanAccessProject : (req, res, next)-> doRequest = (req, res, next) -> getRequestUserAndProject req, res, {allow_auth_token: options?.allow_auth_token}, (err, user, project)-> diff --git a/services/web/app/coffee/router.coffee b/services/web/app/coffee/router.coffee index e704bdcaa5..a6d248224c 100644 --- a/services/web/app/coffee/router.coffee +++ b/services/web/app/coffee/router.coffee @@ -133,7 +133,7 @@ module.exports = class Router app.get '/project/:Project_id/collaborators', SecurityManager.requestCanAccessProject(allow_auth_token: true), CollaboratorsController.getCollaborators app.get '/Project/:Project_id/download/zip', SecurityManager.requestCanAccessProject, ProjectDownloadsController.downloadProject - + app.get '/project/download/zip', SecurityManager.requestCanAccessMultipleProjects, ProjectDownloadsController.downloadMultipleProjects app.get '/tag', AuthenticationController.requireLogin(), TagsController.getAllTags app.post '/project/:project_id/tag', AuthenticationController.requireLogin(), TagsController.processTagsUpdate diff --git a/services/web/test/UnitTests/coffee/Downloads/ProjectDownloadsControllerTests.coffee b/services/web/test/UnitTests/coffee/Downloads/ProjectDownloadsControllerTests.coffee index b17e87d32e..135ff8bfa8 100644 --- a/services/web/test/UnitTests/coffee/Downloads/ProjectDownloadsControllerTests.coffee +++ b/services/web/test/UnitTests/coffee/Downloads/ProjectDownloadsControllerTests.coffee @@ -73,3 +73,50 @@ describe "ProjectDownloadsController", -> .calledWith(sinon.match.any, "downloading project") .should.equal true + describe "downloadMultipleProjects", -> + beforeEach -> + @stream = + pipe: sinon.stub() + @ProjectZipStreamManager.createZipStreamForMultipleProjects = + sinon.stub().callsArgWith(1, null, @stream) + @project_ids = ["project-1", "project-2"] + @req.query = project_ids: @project_ids.join(",") + @res.contentType = sinon.stub() + @res.header = sinon.stub() + @DocumentUpdaterHandler.flushMultipleProjectsToMongo = sinon.stub().callsArgWith(1) + @metrics.inc = sinon.stub() + @ProjectDownloadsController.downloadMultipleProjects @req, @res, @next + + it "should create a zip from the project", -> + @ProjectZipStreamManager.createZipStreamForMultipleProjects + .calledWith(@project_ids) + .should.equal true + + it "should stream the zip to the request", -> + @stream.pipe.calledWith(@res) + .should.equal true + + it "should set the correct content type on the request", -> + @res.contentType + .calledWith("application/zip") + .should.equal true + + it "should flush the projects to mongo", -> + @DocumentUpdaterHandler.flushMultipleProjectsToMongo + .calledWith(@project_ids) + .should.equal true + + it "should name the downloaded file after the project", -> + @res.header + .calledWith( + "Content-Disposition", + "attachment; filename=ShareLaTeX Projects (2 items).zip") + .should.equal true + + it "should record the action via Metrics", -> + @metrics.inc.calledWith("zip-downloads-multiple").should.equal true + + it "should log the action", -> + @logger.log + .calledWith(sinon.match.any, "downloading multiple projects") + .should.equal true diff --git a/services/web/test/UnitTests/coffee/Downloads/ProjectZipStreamManagerTests.coffee b/services/web/test/UnitTests/coffee/Downloads/ProjectZipStreamManagerTests.coffee index ba03d7cbc4..fa8fb5f4e9 100644 --- a/services/web/test/UnitTests/coffee/Downloads/ProjectZipStreamManagerTests.coffee +++ b/services/web/test/UnitTests/coffee/Downloads/ProjectZipStreamManagerTests.coffee @@ -10,13 +10,70 @@ describe "ProjectZipStreamManager", -> beforeEach -> @project_id = "project-id-123" @callback = sinon.stub() - @archive = + @archive = on:-> + append: sinon.stub() @ProjectZipStreamManager = SandboxedModule.require modulePath, requires: "archiver": @archiver = sinon.stub().returns @archive "logger-sharelatex": @logger = {error: sinon.stub(), log: sinon.stub()} "../Project/ProjectEntityHandler" : @ProjectEntityHandler = {} "../FileStore/FileStoreHandler": @FileStoreHandler = {} + "../../models/Project": Project: @Project = {} + + + describe "createZipStreamForMultipleProjects", -> + describe "successfully", -> + beforeEach (done) -> + @project_ids = ["project-1", "project-2"] + @zip_streams = + "project-1": new EventEmitter() + "project-2": new EventEmitter() + + @project_names = + "project-1": "Project One Name" + "project-2": "Project Two Name" + + @ProjectZipStreamManager.createZipStreamForProject = (project_id, callback) => + callback null, @zip_streams[project_id] + setTimeout () => + @zip_streams[project_id].emit "end", + 0 + sinon.spy @ProjectZipStreamManager, "createZipStreamForProject" + + @Project.findById = (project_id, fields, callback) => + callback null, name: @project_names[project_id] + sinon.spy @Project, "findById" + + @ProjectZipStreamManager.createZipStreamForMultipleProjects @project_ids, (args...) => + @callback args... + + @archive.finalize = () -> + done() + + it "should create a zip archive", -> + @archiver.calledWith("zip").should.equal true + + it "should return a stream before any processing is done", -> + @callback.calledWith(sinon.match.falsy, @archive).should.equal true + @callback.calledBefore(@ProjectZipStreamManager.createZipStreamForProject).should.equal true + + it "should get a zip stream for all of the projects", -> + for project_id in @project_ids + @ProjectZipStreamManager.createZipStreamForProject + .calledWith(project_id) + .should.equal true + + it "should get the names of each project", -> + for project_id in @project_ids + @Project.findById + .calledWith(project_id, "name") + .should.equal true + + it "should add all of the projects to the zip", -> + for project_id in @project_ids + @archive.append + .calledWith(@zip_streams[project_id], name: @project_names[project_id] + ".zip") + .should.equal true describe "createZipStreamForProject", -> describe "successfully", -> @@ -89,7 +146,6 @@ describe "ProjectZipStreamManager", -> "/chapters/chapter1.tex": lines: ["chapter1", "content"] @ProjectEntityHandler.getAllDocs = sinon.stub().callsArgWith(1, null, @docs) - @archive.append = sinon.stub() @ProjectZipStreamManager.addAllDocsToArchive @project_id, @archive, (error) => @callback(error) done() @@ -116,7 +172,6 @@ describe "ProjectZipStreamManager", -> "file-id-1" : new EventEmitter() "file-id-2" : new EventEmitter() @ProjectEntityHandler.getAllFiles = sinon.stub().callsArgWith(1, null, @files) - @archive.append = sinon.stub() @FileStoreHandler.getFileStream = (project_id, file_id, {}, callback) => callback null, @streams[file_id] sinon.spy @FileStoreHandler, "getFileStream"